public void ModuleName() {
foreach (var test in new[] {
new { FullName = "abc", Name = "abc", ModuleName = "abc", IsSpecialName = false },
new { FullName = "test.__init__", Name = "__init__", ModuleName = "test", IsSpecialName = true },
new { FullName = "test.__main__", Name = "__main__", ModuleName = "test.__main__", IsSpecialName = true },
new { FullName = "test.support", Name = "support", ModuleName = "test.support", IsSpecialName = false }
}) {
var mp = new ModulePath(test.FullName, string.Empty, string.Empty);
Assert.AreEqual(test.Name, mp.Name);
Assert.AreEqual(test.ModuleName, mp.ModuleName);
Assert.AreEqual(test.IsSpecialName, mp.IsSpecialName);
}
}
public void ModuleIsCompiled() {
foreach (var test in new[] {
new { SourceFile = "abc.py", IsCompiled = false, IsNative = false },
new { SourceFile = "abc.pyc", IsCompiled = true, IsNative = false },
new { SourceFile = "abc.pyo", IsCompiled = true, IsNative = false },
new { SourceFile = "abc.pyd", IsCompiled = true, IsNative = true },
new { SourceFile = "abc.cp35-win_amd64.pyd", IsCompiled = true, IsNative = true },
new { SourceFile = "abc_d.pyd", IsCompiled = true, IsNative = true },
new { SourceFile = "abc_d.cp35-win_amd64.pyd", IsCompiled = true, IsNative = true }
}) {
var mp = new ModulePath(string.Empty, test.SourceFile, string.Empty);
Assert.AreEqual(test.IsCompiled, mp.IsCompiled, test.SourceFile);
Assert.AreEqual(test.IsNative, mp.IsNativeExtension, test.SourceFile);
}
}
private static bool IsTestPackage(ModulePath path) {
var fn = "." + path.FullName;
return fn.Contains(".test.") || fn.Contains(".tests.");
}
private void Parse(bool enqueueOnly, CancellationToken cancel)
{
#if DEBUG
Debug.Assert(Monitor.IsEntered(this));
#endif
var parse = GetCurrentParse();
var tree = parse?.Tree;
var cookie = parse?.Cookie;
if (tree == null)
{
return;
}
var oldParent = MyScope.ParentPackage;
if (FilePath != null)
{
ProjectState.ModulesByFilename[FilePath] = MyScope;
}
if (oldParent != null)
{
// update us in our parent package
oldParent.AddChildPackage(MyScope, _unit);
}
else if (FilePath != null)
{
// we need to check and see if our parent is a package for the case where we're adding a new
// file but not re-analyzing our parent package.
string parentFilename;
if (ModulePath.IsInitPyFile(FilePath))
{
// subpackage
parentFilename = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(FilePath)), "__init__.py");
}
else
{
// just a module
parentFilename = Path.Combine(Path.GetDirectoryName(FilePath), "__init__.py");
}
ModuleInfo parentPackage;
if (ProjectState.ModulesByFilename.TryGetValue(parentFilename, out parentPackage))
{
parentPackage.AddChildPackage(MyScope, _unit);
}
}
_unit = new AnalysisUnit(tree, MyScope.Scope);
AnalysisLog.NewUnit(_unit);
MyScope.Scope.Children = new List <InterpreterScope>();
MyScope.Scope.ClearNodeScopes();
MyScope.Scope.ClearNodeValues();
MyScope.Scope.ClearLinkedVariables();
MyScope.Scope.ClearVariables();
MyScope.ClearUnresolvedModules();
_unit.State.ClearDiagnostics(this);
MyScope.EnsureModuleVariables(_unit.State);
foreach (var value in MyScope.Scope.AllVariables)
{
value.Value.EnqueueDependents();
}
// collect top-level definitions first
var walker = new OverviewWalker(this, _unit, tree);
tree.Walk(walker);
MyScope.Specialize();
// It may be that we have analyzed some child packages of this package already, but because it wasn't analyzed,
// the children were not registered. To handle this possibility, scan analyzed packages for children of this
// package (checked by module name first, then sanity-checked by path), and register any that match.
if (ModulePath.IsInitPyFile(FilePath))
{
string pathPrefix = PathUtils.EnsureEndSeparator(Path.GetDirectoryName(FilePath));
var children =
from pair in ProjectState.ModulesByFilename
// Is the candidate child package in a subdirectory of our package?
let fileName = pair.Key
where fileName.StartsWithOrdinal(pathPrefix, ignoreCase: true)
let moduleName = pair.Value.Name
// Is the full name of the candidate child package qualified with the name of our package?
let lastDot = moduleName.LastIndexOf('.')
where lastDot > 0
let parentModuleName = moduleName.Substring(0, lastDot)
where parentModuleName == MyScope.Name
select pair.Value;
foreach (var child in children)
{
MyScope.AddChildPackage(child, _unit);
}
}
_unit.Enqueue();
if (!enqueueOnly)
{
ProjectState.AnalyzeQueuedEntries(cancel);
}
// publish the analysis now that it's complete/running
Analysis = new ModuleAnalysis(
_unit,
((ModuleScope)_unit.Scope).CloneForPublish()
);
}
/// <summary>
/// Returns a sequence of candidate absolute module names for the given
/// modules.
/// </summary>
/// <param name="importingFromModuleName">
/// The module that is importing the module.
/// </param>
/// <param name="importingFromFilePath">
/// The path to the file that is importing the module.
/// </param>
/// <param name="relativeModuleName">
/// A dotted name identifying the path to the module.
/// </param>
/// <returns>
/// A sequence of strings representing the absolute names of the module
/// in order of precedence.
/// </returns>
internal static IEnumerable <string> ResolvePotentialModuleNames(
string importingFromModuleName,
string importingFromFilePath,
string relativeModuleName,
bool absoluteImports
)
{
string importingFrom = null;
if (!string.IsNullOrEmpty(importingFromModuleName))
{
importingFrom = importingFromModuleName;
if (!string.IsNullOrEmpty(importingFromFilePath) && ModulePath.IsInitPyFile(importingFromFilePath))
{
if (string.IsNullOrEmpty(importingFrom))
{
importingFrom = "__init__";
}
else
{
importingFrom += ".__init__";
}
}
}
if (string.IsNullOrEmpty(relativeModuleName))
{
yield break;
}
// Handle relative module names
if (relativeModuleName.FirstOrDefault() == '.')
{
yield return(GetModuleFullName(importingFrom, relativeModuleName));
yield break;
}
// The two possible names that can be imported here are:
// * relativeModuleName
// * importingFrom.relativeModuleName
// and the order they are returned depends on whether
// absolute_import is enabled or not.
// Assume trailing dots are not part of the import
relativeModuleName = relativeModuleName.TrimEnd('.');
// With absolute_import, we treat the name as complete first.
if (absoluteImports)
{
yield return(relativeModuleName);
}
if (!string.IsNullOrEmpty(importingFrom))
{
var prefix = importingFrom.Split('.');
if (prefix.Length > 1)
{
var adjacentModuleName = string.Join(".", prefix.Take(prefix.Length - 1)) + "." + relativeModuleName;
yield return(adjacentModuleName);
}
}
// Without absolute_import, we treat the name as complete last.
if (!absoluteImports)
{
yield return(relativeModuleName);
}
}
private string GetOutputFile(ModulePath file) {
return PathUtils.GetAbsoluteFilePath(GetOutputDir(file), file.ModuleName + ".idb");
}
public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, ModulePath moduleName, CancellationToken cancel) {
_typeDb = typeDb;
_factory = factory;
_moduleName = moduleName;
_cancel = cancel;
}
foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => ModulePath.GetModulesInPath(p)))
{
public static bool FromBasePathAndFile_NoThrow(
string basePath,
string sourceFile,
out ModulePath modulePath
) => FromBasePathAndFile_NoThrow(basePath, sourceFile, false, out modulePath, out _, out _);
/// <summary>
/// Returns a sequence of candidate absolute module names for the given
/// modules.
/// </summary>
/// <param name="projectEntry">
/// The project entry that is importing the module.
/// </param>
/// <param name="relativeModuleName">
/// A dotted name identifying the path to the module.
/// </param>
/// <returns>
/// A sequence of strings representing the absolute names of the module
/// in order of precedence.
/// </returns>
internal static IEnumerable <string> ResolvePotentialModuleNames(
IPythonProjectEntry projectEntry,
string relativeModuleName,
bool absoluteImports
)
{
string importingFrom = null;
if (projectEntry != null)
{
importingFrom = projectEntry.ModuleName;
if (ModulePath.IsInitPyFile(projectEntry.FilePath))
{
if (string.IsNullOrEmpty(importingFrom))
{
importingFrom = "__init__";
}
else
{
importingFrom += ".__init__";
}
}
}
if (string.IsNullOrEmpty(relativeModuleName))
{
yield break;
}
// Handle relative module names
if (relativeModuleName.FirstOrDefault() == '.')
{
if (string.IsNullOrEmpty(importingFrom))
{
// No source to import relative to.
yield break;
}
var prefix = importingFrom.Split('.');
if (relativeModuleName.LastOrDefault() == '.')
{
// Last part empty means the whole name is dots, so there's
// nothing to concatenate.
yield return(string.Join(".", prefix.Take(prefix.Length - relativeModuleName.Length)));
}
else
{
var suffix = relativeModuleName.Split('.');
var dotCount = suffix.TakeWhile(bit => string.IsNullOrEmpty(bit)).Count();
if (dotCount < prefix.Length)
{
// If we have as many dots as prefix parts, the entire
// name will disappear. Despite what PEP 328 says, in
// reality this means the import will fail.
yield return(string.Join(".", prefix.Take(prefix.Length - dotCount).Concat(suffix.Skip(dotCount))));
}
}
yield break;
}
// The two possible names that can be imported here are:
// * relativeModuleName
// * importingFrom.relativeModuleName
// and the order they are returned depends on whether
// absolute_import is enabled or not.
// With absolute_import, we treat the name as complete first.
if (absoluteImports)
{
yield return(relativeModuleName);
}
if (!string.IsNullOrEmpty(importingFrom))
{
var prefix = importingFrom.Split('.');
if (prefix.Length > 1)
{
var adjacentModuleName = string.Join(".", prefix.Take(prefix.Length - 1)) + "." + relativeModuleName;
yield return(adjacentModuleName);
}
}
// Without absolute_import, we treat the name as complete last.
if (!absoluteImports)
{
yield return(relativeModuleName);
}
}
public static string PathToModuleName(string path)
{
return(ModulePath.FromFullPath(path).ModuleName);
}
internal static bool FromBasePathAndFile_NoThrow(
string basePath,
string sourceFile,
out ModulePath modulePath
) => FromBasePathAndFile_NoThrow(basePath, sourceFile, null, out modulePath, out _, out _);
internal static bool FromBasePathAndName_NoThrow(
string basePath,
string moduleName,
out ModulePath modulePath
) => FromBasePathAndName_NoThrow(basePath, moduleName, null, null, out modulePath, out _, out _, out _);
private string GenerateDbFile(IPythonInterpreterFactory interpreter, ModulePath moduleName, List<string> existingModules, string dbFile, FileStream fs) {
// we need to generate the DB file
dbFile = Path.Combine(ReferencesDatabasePath, moduleName + ".$project.idb");
int retryCount = 0;
while (File.Exists(dbFile)) {
dbFile = Path.Combine(ReferencesDatabasePath, moduleName + "." + ++retryCount + ".$project.idb");
}
var args = new List<string> {
PythonToolsInstallPath.GetFile("ExtensionScraper.py"),
"scrape",
};
if (moduleName.IsNativeExtension) {
args.Add("-");
args.Add(moduleName.SourceFile);
} else {
args.Add(moduleName.ModuleName);
args.Add(moduleName.LibraryPath);
}
args.Add(Path.ChangeExtension(dbFile, null));
using (var output = interpreter.Run(args.ToArray())) {
if (_cancel.CanBeCanceled) {
if (WaitHandle.WaitAny(new[] { _cancel.WaitHandle, output.WaitHandle }) != 1) {
// we were cancelled
return null;
}
} else {
output.Wait();
}
if (output.ExitCode == 0) {
// [FileName]|interpGuid|interpVersion|DateTimeStamp|[db_file.idb]
// save the new entry in the DB file
existingModules.Add(
String.Format("{0}|{1}|{2}|{3}",
moduleName.SourceFile,
interpreter.Configuration.Id,
new FileInfo(moduleName.SourceFile).LastWriteTime.ToString("O"),
dbFile
)
);
fs.Seek(0, SeekOrigin.Begin);
fs.SetLength(0);
using (var sw = new StreamWriter(fs)) {
sw.Write(String.Join(Environment.NewLine, existingModules));
sw.Flush();
}
} else {
throw new CannotAnalyzeExtensionException(string.Join(Environment.NewLine, output.StandardErrorLines));
}
}
return dbFile;
}
public PythonTestCase(PythonProjectSettings settings, TestCase testCase, PythonDebugMode debugMode) {
Settings = settings;
TestCase = testCase;
TestAnalyzer.ParseFullyQualifiedTestName(
testCase.FullyQualifiedName,
out TestFile,
out TestClass,
out TestMethod
);
TestFilePath = CommonUtils.GetAbsoluteFilePath(Settings.ProjectHome, TestFile);
ModulePath = ModulePath.FromFullPath(TestFilePath);
WorkingDirectory = Settings.WorkingDir;
var paths = settings.SearchPath.ToList();
paths.Insert(0, ModulePath.LibraryPath);
paths.Insert(0, WorkingDirectory);
if (debugMode == PythonDebugMode.PythonOnly) {
paths.Insert(0, PtvsdSearchPath);
}
SearchPaths = string.Join(
";",
paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase)
);
var arguments = new List<string> {
TestLauncherPath,
"-m", ModulePath.ModuleName,
"-t", string.Format("{0}.{1}", TestClass, TestMethod)
};
if (debugMode == PythonDebugMode.PythonOnly) {
var secretBuffer = new byte[24];
RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer);
DebugSecret = Convert.ToBase64String(secretBuffer)
.Replace('+', '-')
.Replace('/', '_')
.TrimEnd('=');
DebugPort = GetFreePort();
arguments.AddRange(new[] {
"-s", DebugSecret,
"-p", DebugPort.ToString()
});
} else if (debugMode == PythonDebugMode.PythonAndNative) {
arguments.Add("-x");
}
Arguments = arguments;
Environment = new Dictionary<string, string>();
if (!string.IsNullOrEmpty(settings.DjangoSettingsModule)) {
Environment["DJANGO_SETTINGS_MODULE"] = settings.DjangoSettingsModule;
}
foreach (var envVar in settings.Environment) {
Environment[envVar.Key] = envVar.Value;
}
}
internal static bool FromBasePathAndFile_NoThrow(
string basePath,
string sourceFile,
bool isPackage,
out ModulePath modulePath,
out bool isInvalid,
out bool isMissing
)
{
modulePath = default(ModulePath);
isInvalid = false;
isMissing = false;
if (!PathEqualityComparer.Instance.StartsWith(sourceFile, basePath, allowFullMatch: false))
{
return(false);
}
var nameMatch = PythonFileRegex.Match(PathUtils.GetFileName(sourceFile));
if (!nameMatch.Success)
{
isInvalid = true;
return(false);
}
var bits = new List <string> {
nameMatch.Groups["name"].Value
};
var path = PathUtils.TrimEndSeparator(PathUtils.GetParent(sourceFile));
var lastWasStubs = false;
while (PathEqualityComparer.Instance.StartsWith(path, basePath, allowFullMatch: false))
{
if (!isPackage && string.IsNullOrEmpty(GetPackageInitPy(path)))
{
isMissing = true;
return(false);
}
if (lastWasStubs)
{
isInvalid = true;
return(false);
}
var bit = PathUtils.GetFileName(path);
var m = PythonPackageRegex.Match(bit);
if (!m.Success)
{
isInvalid = true;
return(false);
}
lastWasStubs = m.Groups["stubs"].Success;
bits.Add(PathUtils.GetFileName(path));
path = PathUtils.TrimEndSeparator(PathUtils.GetParent(path));
}
if (!PathEqualityComparer.Instance.Equals(basePath, path))
{
isMissing = true;
return(false);
}
var moduleName = string.Join(".", bits.AsEnumerable().Reverse());
modulePath = new ModulePath(moduleName, sourceFile, basePath);
return(true);
}
private string GetOutputDir(ModulePath file) {
if (_treatPathsAsStandardLibrary.Contains(file.LibraryPath)) {
return _outDir;
} else {
return Path.Combine(_outDir, Regex.Replace(
PathUtils.TrimEndSeparator(PathUtils.GetFileOrDirectoryName(file.LibraryPath)),
@"[.\\/\s]",
"_"
));
}
}
internal static bool FromBasePathAndName_NoThrow(
string basePath,
string moduleName,
Func <string, bool> isPackage,
Func <string, string, string> getModule,
out ModulePath modulePath,
out bool isInvalid,
out bool isMissing,
out string errorParameter
)
{
modulePath = default(ModulePath);
isInvalid = false;
isMissing = false;
errorParameter = null;
var bits = moduleName.Split('.');
var lastBit = bits.Last();
if (isPackage == null)
{
isPackage = f => !string.IsNullOrEmpty(GetPackageInitPy(f));
}
if (getModule == null)
{
getModule = (dir, mod) => {
var pack = GetPackageInitPy(Path.Combine(dir, mod));
if (File.Exists(pack))
{
return(pack);
}
var mods = PathUtils.EnumerateFiles(dir, mod + "*", recurse: false).ToArray();
return(mods.FirstOrDefault(p => IsModuleNameMatch(PythonStubRegex, p, mod)) ??
mods.FirstOrDefault(p => IsModuleNameMatch(PythonBinaryRegex, p, mod)) ??
mods.FirstOrDefault(p => IsModuleNameMatch(PythonFileRegex, p, mod)));
};
}
var path = basePath;
foreach (var bit in bits.Take(bits.Length - 1))
{
if (!PythonPackageRegex.IsMatch(bit))
{
isInvalid = true;
errorParameter = bit;
return(false);
}
if (string.IsNullOrEmpty(path))
{
path = bit;
}
else
{
path = Path.Combine(path, bit);
}
if (!isPackage(path))
{
isMissing = true;
errorParameter = path;
return(false);
}
}
if (!PythonPackageRegex.IsMatch(lastBit))
{
isInvalid = true;
errorParameter = moduleName;
return(false);
}
path = getModule(path, lastBit);
if (string.IsNullOrEmpty(path))
{
isMissing = true;
errorParameter = moduleName;
return(false);
}
modulePath = new ModulePath(moduleName, path, basePath);
return(true);
}
public AnalysisItem(ModulePath path) {
_path = path;
}
/// <summary>
/// Asynchrously loads the specified extension module into the type
/// database making the completions available.
///
/// If the module has not already been analyzed it will be analyzed and
/// then loaded.
///
/// If the specified module was already loaded it replaces the existing
/// module.
///
/// Returns a new Task which can be blocked upon until the analysis of
/// the new extension module is available.
///
/// If the extension module cannot be analyzed an exception is reproted.
/// </summary>
/// <param name="cancellationToken">A cancellation token which can be
/// used to cancel the async loading of the module</param>
/// <param name="extensionModuleFilename">The filename of the extension
/// module to be loaded</param>
/// <param name="interpreter">The Python interprefer which will be used
/// to analyze the extension module.</param>
/// <param name="moduleName">The module name of the extension module.</param>
public void LoadExtensionModule(
ModulePath moduleName,
CancellationToken cancellationToken = default(CancellationToken)
) {
var loader = new ExtensionModuleLoader(
this,
_factory,
moduleName,
cancellationToken
);
loader.LoadExtensionModule();
}
