private IEnumerable<string> GetAssemblyFiles(string moduleName)
{
ModulePath modulePath = new ModulePath(moduleName);
ModuleItemPath moduleBinPath = new ModuleItemPath(moduleName, "Bin");
if (Directory.Exists(moduleBinPath.PhysicalPath))
{
return Directory.EnumerateFiles(moduleBinPath.PhysicalPath);
}
else
{
return new string[0];
}
}
// Avoid hitting the filesystem, but exclude non-importable
// paths. Ideally, we'd stop at the first path that's a known
// search path, except we don't know search paths here.
private static bool IsPackageCheck(string path)
{
return(ModulePath.IsImportable(PathUtils.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar))));
}
private static string GetCoveragePath(IEnumerable <TestCase> tests)
{
string bestFile = null, bestClass = null, bestMethod = null;
// Try and generate a friendly name for the coverage report. We use
// the filename, class, and method. We include each one if we're
// running from a single filename/class/method. When we have multiple
// we drop the identifying names. If we have multiple files we
// go to the top level directory... If all else fails we do "pycov".
foreach (var test in tests)
{
string testFile, testClass, testMethod;
TestReader.ParseFullyQualifiedTestName(
test.FullyQualifiedName,
out testFile,
out testClass,
out testMethod
);
bestFile = UpdateBestFile(bestFile, test.CodeFilePath);
if (bestFile != test.CodeFilePath)
{
// Different files, don't include class/methods even
// if they happen to be the same.
bestClass = bestMethod = "";
}
bestClass = UpdateBest(bestClass, testClass);
bestMethod = UpdateBest(bestMethod, testMethod);
}
string filename = "";
if (!String.IsNullOrWhiteSpace(bestFile))
{
if (ModulePath.IsPythonSourceFile(bestFile))
{
filename = ModulePath.FromFullPath(bestFile).ModuleName;
}
else
{
filename = Path.GetFileName(bestFile);
}
}
else
{
filename = "pycov";
}
if (!String.IsNullOrWhiteSpace(bestClass))
{
filename += "_" + bestClass;
}
if (!String.IsNullOrWhiteSpace(bestMethod))
{
filename += "_" + bestMethod;
}
filename += "_" + DateTime.Now.ToString("s").Replace(':', '_');
return(Path.Combine(Path.GetTempPath(), filename));
}
internal PythonAnalyzer AnalyzeDir(string dir, PythonLanguageVersion version = PythonLanguageVersion.V27, IEnumerable <string> excludeDirectories = null, CancellationToken?cancel = null)
{
List <string> files = new List <string>();
try {
ISet <string> excluded = null;
if (excludeDirectories != null)
{
excluded = new HashSet <string>(excludeDirectories, StringComparer.InvariantCultureIgnoreCase);
}
CollectFiles(dir, files, excluded);
} catch (DirectoryNotFoundException) {
return(null);
}
List <FileStreamReader> sourceUnits = new List <FileStreamReader>();
foreach (string file in files)
{
sourceUnits.Add(
new FileStreamReader(file)
);
}
Stopwatch sw = new Stopwatch();
sw.Start();
long start0 = sw.ElapsedMilliseconds;
// Explicitly specify the builtins name because we are using a 2.7
// interpreter for all versions.
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
var projectState = new PythonAnalyzer(fact, fact.CreateInterpreter(), "__builtin__");
projectState.ReloadModulesAsync().WaitAndUnwrapExceptions();
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
var modules = new List <IPythonProjectEntry>();
foreach (var sourceUnit in sourceUnits)
{
try {
modules.Add(projectState.AddModule(
ModulePath.FromFullPath(sourceUnit.Path).ModuleName,
sourceUnit.Path,
null
));
} catch (ArgumentException) {
// Invalid module name, so skip the module
}
}
long start1 = sw.ElapsedMilliseconds;
Trace.TraceInformation("AddSourceUnit: {0} ms", start1 - start0);
var nodes = new List <Microsoft.PythonTools.Parsing.Ast.PythonAst>();
for (int i = 0; i < modules.Count; i++)
{
PythonAst ast = null;
try {
var sourceUnit = sourceUnits[i];
ast = Parser.CreateParser(sourceUnit, version).ParseFile();
} catch (Exception) {
}
nodes.Add(ast);
}
long start2 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Parse: {0} ms", start2 - start1);
for (int i = 0; i < modules.Count; i++)
{
var ast = nodes[i];
if (ast != null)
{
modules[i].UpdateTree(ast, null);
}
}
long start3 = sw.ElapsedMilliseconds;
for (int i = 0; i < modules.Count; i++)
{
Trace.TraceInformation("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path);
var ast = nodes[i];
if (ast != null)
{
modules[i].Analyze(cancel ?? CancellationToken.None, true);
}
}
if (modules.Count > 0)
{
Trace.TraceInformation("Analyzing queue");
modules[0].AnalysisGroup.AnalyzeQueuedEntries(cancel ?? CancellationToken.None);
}
long start4 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Analyze: {0} ms", start4 - start3);
return(projectState);
}
/// <summary>
/// For Saving Module After Decrypting
/// </summary>
public void Save()
{
if (Module.IsILOnly)
{
var Options = new ModuleWriterOptions(Module)
{
Logger = DummyLogger.NoThrowInstance
};
var NewPath = Module.Kind.Equals(ModuleKind.Dll) ? ModulePath.Replace(".dll", "-Decrypted.dll") : ModulePath.Replace(".exe", "-Decrypted.exe");
Module.Write(NewPath, Options);
Log.Information($"Module Saved : {NewPath}");
System.Console.ReadKey();
}
else
{
var Options = new NativeModuleWriterOptions(Module, false)
{
Logger = DummyLogger.NoThrowInstance
};
var NewPath = Module.Kind.Equals(ModuleKind.Dll) ? ModulePath.Replace(".dll", "-Decrypted.dll") : ModulePath.Replace(".exe", "-Decrypted.exe");
Module.NativeWrite(NewPath, Options);
Log.Information($"Module Saved : {NewPath}");
System.Console.ReadKey();
}
}
private Task <IProjectEntry> AddFileAsync(Uri documentUri, Uri fromSearchPath, IAnalysisCookie cookie = null)
{
var item = _projectFiles.GetEntry(documentUri, throwIfMissing: false);
if (item != null)
{
return(Task.FromResult(item));
}
IEnumerable <string> aliases = null;
var path = GetLocalPath(documentUri);
if (fromSearchPath != null)
{
if (ModulePath.FromBasePathAndFile_NoThrow(GetLocalPath(fromSearchPath), path, out var mp))
{
aliases = new[] { mp.ModuleName };
}
}
else
{
aliases = GetImportNames(documentUri).Select(mp => mp.ModuleName).ToArray();
}
if (!(aliases?.Any() ?? false))
{
aliases = new[] { Path.GetFileNameWithoutExtension(path) };
}
var reanalyzeEntries = aliases.SelectMany(a => _analyzer.GetEntriesThatImportModule(a, true)).ToArray();
var first = aliases.FirstOrDefault();
var pyItem = _analyzer.AddModule(first, path, documentUri, cookie);
item = pyItem;
foreach (var a in aliases.Skip(1))
{
_analyzer.AddModuleAlias(first, a);
}
var actualItem = _projectFiles.GetOrAddEntry(documentUri, item);
if (actualItem != item)
{
return(Task.FromResult(actualItem));
}
if (_clientCaps?.python?.analysisUpdates ?? false)
{
pyItem.OnNewAnalysis += ProjectEntry_OnNewAnalysis;
}
if (item is IDocument doc)
{
EnqueueItem(doc);
}
if (reanalyzeEntries != null)
{
foreach (var entryRef in reanalyzeEntries)
{
_queue.Enqueue(entryRef, AnalysisPriority.Low);
}
}
return(Task.FromResult(item));
}
/// <summary>
/// Determines whether the specified directory is an importable package.
/// </summary>
public bool IsPackage(string directory)
{
return(ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ?
!string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) :
Directory.Exists(directory));
}
private void AstNativeBuiltinScrape(PythonVersion version) {
AstScrapedPythonModule.KeepAst = true;
version.AssertInstalled();
using (var analysis = CreateAnalysis(version)) {
try {
var fact = (AstPythonInterpreterFactory)analysis.Analyzer.InterpreterFactory;
var interp = (AstPythonInterpreter)analysis.Analyzer.Interpreter;
var ctxt = interp.CreateModuleContext();
var dllsDir = PathUtils.GetAbsoluteDirectoryPath(fact.Configuration.PrefixPath, "DLLs");
if (!Directory.Exists(dllsDir)) {
Assert.Inconclusive("Configuration does not have DLLs");
}
var report = new List<string>();
var permittedImports = fact.GetLanguageVersion().Is2x() ?
new[] { interp.BuiltinModuleName, "exceptions" } :
new[] { interp.BuiltinModuleName };
foreach (var pyd in PathUtils.EnumerateFiles(dllsDir, "*", recurse: false).Where(ModulePath.IsPythonFile)) {
var mp = ModulePath.FromFullPath(pyd);
if (mp.IsDebug) {
continue;
}
Console.WriteLine("Importing {0} from {1}", mp.ModuleName, mp.SourceFile);
var mod = interp.ImportModule(mp.ModuleName);
Assert.IsInstanceOfType(mod, typeof(AstScrapedPythonModule));
mod.Imported(ctxt);
var modPath = fact.GetCacheFilePath(pyd);
Assert.IsTrue(File.Exists(modPath), "No cache file created");
_moduleCache = File.ReadAllText(modPath);
var errors = ((AstScrapedPythonModule)mod).ParseErrors ?? Enumerable.Empty<string>();
foreach (var err in errors) {
Console.WriteLine(err);
}
Assert.AreEqual(0, errors.Count(), "Parse errors occurred");
var ast = ((AstScrapedPythonModule)mod).Ast;
var imports = ((Ast.SuiteStatement)ast.Body).Statements
.OfType<Ast.ImportStatement>()
.SelectMany(s => s.Names)
.Select(n => n.MakeString())
.Except(permittedImports)
.ToArray();
// We expect no imports (after excluding builtins)
report.AddRange(imports.Select(n => $"{mp.ModuleName} imported {n}"));
_moduleCache = null;
}
AssertUtil.ContainsExactly(report);
} finally {
_analysisLog = analysis.GetLogContent(CultureInfo.InvariantCulture);
}
}
}
private async Task <IProjectEntry> AddFileAsync(Uri documentUri, Uri fromSearchPath, IAnalysisCookie cookie = null)
{
var item = ProjectFiles.GetEntry(documentUri, throwIfMissing: false);
if (item != null)
{
return(item);
}
string[] aliases = null;
var path = GetLocalPath(documentUri);
if (fromSearchPath != null)
{
if (ModulePath.FromBasePathAndFile_NoThrow(GetLocalPath(fromSearchPath), path, out var mp))
{
aliases = new[] { mp.ModuleName };
}
}
else
{
aliases = GetImportNames(documentUri).Select(mp => mp.ModuleName).ToArray();
}
if (aliases.IsNullOrEmpty())
{
aliases = new[] { Path.GetFileNameWithoutExtension(path) };
}
var reanalyzeEntries = aliases.SelectMany(a => Analyzer.GetEntriesThatImportModule(a, true)).ToArray();
var first = aliases.FirstOrDefault();
var pyItem = Analyzer.AddModule(first, path, documentUri, cookie);
item = pyItem;
foreach (var a in aliases.Skip(1))
{
Analyzer.AddModuleAlias(first, a);
}
var actualItem = ProjectFiles.GetOrAddEntry(documentUri, item);
if (actualItem != item)
{
return(actualItem);
}
pyItem.OnNewAnalysis += (o, e) => OnPythonEntryNewAnalysis(pyItem);
if (item is IDocument doc)
{
EnqueueItem(doc);
}
foreach (var entryRef in reanalyzeEntries)
{
_queue.Enqueue(entryRef, AnalysisPriority.Low);
}
return(item);
}
public void Imported(IModuleContext context)
{
if (_scraped)
{
return;
}
_scraped = true;
var interp = context as AstPythonInterpreter;
var fact = interp?.Factory;
if (fact == null || !File.Exists(fact.Configuration.InterpreterPath))
{
return;
}
ModulePath mp = AstPythonInterpreterFactory.FindModule(fact, _filePath);
if (string.IsNullOrEmpty(mp.FullName))
{
return;
}
var sm = PythonToolsInstallPath.TryGetFile("scrape_module.py", GetType().Assembly);
if (!File.Exists(sm))
{
return;
}
Stream code = null;
using (var p = ProcessOutput.RunHiddenAndCapture(
fact.Configuration.InterpreterPath, "-E", sm, mp.LibraryPath, mp.ModuleName
)) {
p.Wait();
if (p.ExitCode == 0)
{
var ms = new MemoryStream();
code = ms;
using (var sw = new StreamWriter(ms, Encoding.UTF8, 4096, true)) {
foreach (var line in p.StandardOutputLines)
{
sw.WriteLine(line);
}
}
}
}
if (code == null)
{
return;
}
PythonAst ast;
code.Seek(0, SeekOrigin.Begin);
using (var sr = new StreamReader(code, Encoding.UTF8))
using (var parser = Parser.CreateParser(sr, fact.GetLanguageVersion())) {
ast = parser.ParseFile();
}
lock (_members) {
var walker = new AstAnalysisWalker(interp, ast, this, _filePath, _members, false);
ast.Walk(walker);
}
}
// Avoid hitting the filesystem, but exclude non-importable
// paths. Ideally, we'd stop at the first path that's a known
// search path, except we don't know search paths here.
private static bool IsPackageCheck(string path)
{
return(ModulePath.IsImportable(PathUtils.GetFileOrDirectoryName(path)));
}
private void ConectionReceivedEvent(object sender, EventReceivedEventArgs e)
{
switch (e.Name)
{
case TP.ResultEvent.Name:
var result = (TP.ResultEvent)e.Event;
TestOutcome outcome = TestOutcome.None;
switch (result.outcome)
{
case "passed": outcome = TestOutcome.Passed; break;
case "failed": outcome = TestOutcome.Failed; break;
case "skipped": outcome = TestOutcome.Skipped; break;
}
var testResult = new TestResult(_curTest);
RecordEnd(
_frameworkHandle,
_curTest,
testResult,
_stdOut.ToString(),
_stdErr.ToString(),
outcome,
result
);
_stdOut.Clear();
_stdErr.Clear();
break;
case TP.StartEvent.Name:
var start = (TP.StartEvent)e.Event;
_curTest = null;
foreach (var test in _tests)
{
string testFile, testClass, testMethod;
TestReader.ParseFullyQualifiedTestName(
test.FullyQualifiedName,
out testFile,
out testClass,
out testMethod
);
string testFilePath = PathUtils.GetAbsoluteFilePath(_settings.ProjectHome, test.CodeFilePath);
var modulePath = ModulePath.FromFullPath(testFilePath);
if (start.test == modulePath.ModuleName + "." + testClass + "." + testMethod)
{
_curTest = test;
break;
}
}
if (_curTest != null)
{
_frameworkHandle.RecordStart(_curTest);
}
else
{
Warning(Strings.Test_UnexpectedResult.FormatUI(start.classname, start.method));
}
break;
case TP.StdErrEvent.Name:
var err = (TP.StdErrEvent)e.Event;
_stdErr.Append(err.content);
break;
case TP.StdOutEvent.Name:
var outp = (TP.StdOutEvent)e.Event;
_stdOut.Append(outp.content);
break;
case TP.DoneEvent.Name:
_done.Set();
break;
}
}
internal void RemoveModuleArea(string moduleName)
{
ModulePath modulePath = new ModulePath(moduleName);
Kooboo.IO.IOUtility.DeleteDirectory(modulePath.PhysicalPath, true);
}
/// <summary> /// Determines whether the specified directory is an importable package. /// </summary> private bool IsPackage(string directory) => ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ? !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : _fs.DirectoryExists(directory);
private async Task FullStdLibTest(InterpreterConfiguration configuration, params string[] skipModules)
{
configuration.AssertInstalled();
var moduleUri = TestData.GetDefaultModuleUri();
var moduleDirectory = Path.GetDirectoryName(moduleUri.LocalPath);
var services = await CreateServicesAsync(moduleDirectory, configuration);
var interpreter = services.GetService <IPythonInterpreter>();
var modules = ModulePath.GetModulesInLib(configuration.LibraryPath, configuration.SitePackagesPath).ToList();
var skip = new HashSet <string>(skipModules);
skip.UnionWith(new[] {
@"matplotlib.backends._backend_gdk",
@"matplotlib.backends._backend_gtkagg",
@"matplotlib.backends._gtkagg",
"test.test_pep3131",
"test.test_unicode_identifiers",
"test.test_super" // nonlocal syntax error
});
skip.UnionWith(modules.Select(m => m.FullName)
.Where(n => n.StartsWith(@"test.badsyntax") || n.StartsWith("test.bad_coding")));
var anySuccess = false;
var anyExtensionSuccess = false;
var anyExtensionSeen = false;
var anyParseError = false;
foreach (var m in skip)
{
((MainModuleResolution)interpreter.ModuleResolution).AddUnimportableModule(m);
}
var set = modules
.Where(m => !skip.Contains(m.ModuleName))
.GroupBy(m => {
var i = m.FullName.IndexOf('.');
return(i <= 0 ? m.FullName : m.FullName.Remove(i));
})
.SelectMany(g => g.Select(m => Tuple.Create(m, m.ModuleName)))
.ToArray();
set = set.Where(x => x.Item2 != null && x.Item2.Contains("grammar")).ToArray();
var sb = new StringBuilder();
foreach (var r in set)
{
var module = interpreter.ModuleResolution.GetOrLoadModule(r.Item2);
if (module != null)
{
sb.AppendLine($"import {module.Name}");
}
}
await GetAnalysisAsync(sb.ToString(), services);
foreach (var r in set)
{
var modName = r.Item1;
var mod = interpreter.ModuleResolution.GetOrLoadModule(r.Item2);
anyExtensionSeen |= modName.IsNativeExtension;
switch (mod)
{
case null:
Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile);
break;
case CompiledPythonModule compiledPythonModule:
var errors = compiledPythonModule.GetParseErrors().ToArray();
if (errors.Any())
{
anyParseError = true;
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
foreach (var e in errors)
{
Trace.TraceError(e.Message);
}
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
}
break;
case IPythonModule _: {
var filteredErrors = ((IDocument)mod).GetParseErrors().Where(e => !e.Message.Contains("encoding problem")).ToArray();
if (filteredErrors.Any())
{
// Do not fail due to errors in installed packages
if (!mod.FilePath.Contains("site-packages"))
{
anyParseError = true;
}
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
foreach (var e in filteredErrors)
{
Trace.TraceError(e.Message);
}
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
}
break;
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules");
Assert.IsFalse(anyParseError, "parse errors occurred");
}
public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, ModulePath moduleName, CancellationToken cancel)
{
_typeDb = typeDb;
_factory = factory ?? throw new ArgumentNullException(nameof(factory));
_moduleName = moduleName;
_cancel = cancel;
}
private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configuration)
{
configuration.AssertInstalled();
var moduleUri = TestData.GetDefaultModuleUri();
var moduleDirectory = Path.GetDirectoryName(moduleUri.LocalPath);
var services = await CreateServicesAsync(moduleDirectory, configuration);
var interpreter = services.GetService <IPythonInterpreter>();
// TODO: this is Windows only
var dllsDir = Path.Combine(Path.GetDirectoryName(interpreter.Configuration.LibraryPath), "DLLs");
if (!Directory.Exists(dllsDir))
{
Assert.Inconclusive("Configuration does not have DLLs");
}
var report = new List <string>();
var permittedImports = interpreter.LanguageVersion.Is2x() ?
new[] { interpreter.ModuleResolution.BuiltinModuleName, "exceptions" } :
new[] { interpreter.ModuleResolution.BuiltinModuleName };
var modules = new List <IPythonModule>();
foreach (var pyd in PathUtils.EnumerateFiles(dllsDir, "*", recurse: false).Where(ModulePath.IsPythonFile))
{
var mp = ModulePath.FromFullPath(pyd);
if (mp.IsDebug)
{
continue;
}
Console.WriteLine(@"Importing {0} from {1}", mp.ModuleName, mp.SourceFile);
modules.Add(interpreter.ModuleResolution.GetOrLoadModule(mp.ModuleName));
}
foreach (var mod in modules)
{
Assert.IsInstanceOfType(mod, typeof(CompiledPythonModule));
await((ModuleCache)interpreter.ModuleResolution.ModuleCache).CacheWritingTask;
var modPath = interpreter.ModuleResolution.ModuleCache.GetCacheFilePath(mod.FilePath);
Assert.IsTrue(File.Exists(modPath), "No cache file created");
var doc = (IDocument)mod;
var ast = await doc.GetAstAsync();
var errors = doc.GetParseErrors().ToArray();
foreach (var err in errors)
{
Console.WriteLine(err);
}
Assert.AreEqual(0, errors.Length, "Parse errors occurred");
var imports = ((SuiteStatement)ast.Body).Statements
.OfType <ImportStatement>()
.SelectMany(s => s.Names)
.Select(n => n.MakeString())
.Except(permittedImports)
.ToArray();
// We expect no imports (after excluding builtins)
report.AddRange(imports.Select(n => $"{mod.Name} imported {n}"));
}
report.Should().BeEmpty();
}
public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, ModulePath moduleName, CancellationToken cancel)
{
_typeDb = typeDb;
_factory = factory;
_moduleName = moduleName;
_cancel = cancel;
}
private async Task FullStdLibTest(PythonVersion v, params string[] skipModules) {
v.AssertInstalled();
var factory = new AstPythonInterpreterFactory(v.Configuration, new InterpreterFactoryCreationOptions {
DatabasePath = TestData.GetTempPath(),
UseExistingCache = false
});
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
var skip = new HashSet<string>(skipModules);
skip.UnionWith(new[] {
"matplotlib.backends._backend_gdk",
"matplotlib.backends._backend_gtkagg",
"matplotlib.backends._gtkagg",
});
bool anySuccess = false;
bool anyExtensionSuccess = false, anyExtensionSeen = false;
bool anyParseError = false;
using (var analyzer = new PythonAnalysis(factory)) {
try {
var tasks = new List<Task<Tuple<ModulePath, IPythonModule>>>();
var interp = (AstPythonInterpreter)analyzer.Analyzer.Interpreter;
foreach (var m in skip) {
interp.AddUnimportableModule(m);
}
foreach (var r in modules
.Where(m => !skip.Contains(m.ModuleName))
.GroupBy(m => {
int i = m.FullName.IndexOf('.');
return i <= 0 ? m.FullName : m.FullName.Remove(i);
})
.AsParallel()
.SelectMany(g => g.Select(m => Tuple.Create(m, interp.ImportModule(m.ModuleName))))
) {
var modName = r.Item1;
var mod = r.Item2;
anyExtensionSeen |= modName.IsNativeExtension;
if (mod == null) {
Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile);
} else if (mod is AstScrapedPythonModule smod) {
if (smod.ParseErrors?.Any() ?? false) {
anyParseError = true;
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
foreach (var e in smod.ParseErrors) {
Trace.TraceError(e);
}
} else {
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
} else if (mod is AstPythonModule) {
// pass
} else {
Trace.TraceError("imported {0} as type {1}", modName.ModuleName, mod.GetType().FullName);
}
}
} finally {
_analysisLog = analyzer.GetLogContent(CultureInfo.InvariantCulture);
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules");
Assert.IsFalse(anyParseError, "parse errors occurred");
}
private string GenerateDbFile(IPythonInterpreterFactory interpreter, ModulePath moduleName, List <string> existingModules, string dbFile, FileStream fs)
{
// we need to generate the DB file
dbFile = Path.Combine(ReferencesDatabasePath, moduleName + ".$project.idb");
int retryCount = 0;
while (File.Exists(dbFile))
{
dbFile = Path.Combine(ReferencesDatabasePath, moduleName + "." + ++retryCount + ".$project.idb");
}
var args = new List <string> {
PythonToolsInstallPath.GetFile("ExtensionScraper.py"),
"scrape",
};
if (moduleName.IsNativeExtension)
{
args.Add("-");
args.Add(moduleName.SourceFile);
}
else
{
args.Add(moduleName.ModuleName);
args.Add(moduleName.LibraryPath);
}
args.Add(Path.ChangeExtension(dbFile, null));
using (var output = interpreter.Run(args.ToArray())) {
if (_cancel.CanBeCanceled)
{
if (WaitHandle.WaitAny(new[] { _cancel.WaitHandle, output.WaitHandle }) != 1)
{
// we were cancelled
return(null);
}
}
else
{
output.Wait();
}
if (output.ExitCode == 0)
{
// [FileName]|interpGuid|interpVersion|DateTimeStamp|[db_file.idb]
// save the new entry in the DB file
existingModules.Add(
String.Format("{0}|{1}|{2}|{3}",
moduleName.SourceFile,
interpreter.Configuration.Id,
new FileInfo(moduleName.SourceFile).LastWriteTime.ToString("O"),
dbFile
)
);
fs.Seek(0, SeekOrigin.Begin);
fs.SetLength(0);
using (var sw = new StreamWriter(fs)) {
sw.Write(String.Join(Environment.NewLine, existingModules));
sw.Flush();
}
}
else
{
throw new CannotAnalyzeExtensionException(string.Join(Environment.NewLine, output.StandardErrorLines));
}
}
return(dbFile);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
var buildEngine = new MSBuild.ProjectCollection();
try {
// Load all the test containers passed in (.pyproj msbuild files)
foreach (string source in sources)
{
buildEngine.LoadProject(source);
}
foreach (var proj in buildEngine.LoadedProjects)
{
using (var provider = new MSBuildProjectInterpreterFactoryProvider(_interpreterService, proj)) {
try {
provider.DiscoverInterpreters();
} catch (InvalidDataException) {
// This exception can be safely ignored here.
}
var factory = provider.ActiveInterpreter;
if (factory == _interpreterService.NoInterpretersValue)
{
if (logger != null)
{
logger.SendMessage(TestMessageLevel.Warning, "No interpreters available for project " + proj.FullPath);
}
continue;
}
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, proj.GetPropertyValue(PythonConstants.ProjectHomeSetting) ?? "."));
// Do the analysis even if the database is not up to date. At
// worst, we'll get no results.
using (var analyzer = new TestAnalyzer(
factory,
proj.FullPath,
projectHome,
TestExecutor.ExecutorUri
)) {
// Provide all files to the test analyzer
foreach (var item in proj.GetItems("Compile"))
{
string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
string fullName;
try {
fullName = ModulePath.FromFullPath(fileAbsolutePath).ModuleName;
} catch (ArgumentException) {
if (logger != null)
{
logger.SendMessage(TestMessageLevel.Warning, "File has an invalid module name: " + fileAbsolutePath);
}
continue;
}
try {
using (var reader = new StreamReader(fileAbsolutePath)) {
analyzer.AddModule(fullName, fileAbsolutePath, reader);
}
} catch (FileNotFoundException) {
// user deleted file, we send the test update, but the project
// isn't saved.
#if DEBUG
} catch (Exception ex) {
if (logger != null)
{
logger.SendMessage(TestMessageLevel.Warning, "Failed to discover tests in " + fileAbsolutePath);
logger.SendMessage(TestMessageLevel.Informational, ex.ToString());
}
}
#else
} catch (Exception) {
if (logger != null)
{
logger.SendMessage(TestMessageLevel.Warning, "Failed to discover tests in " + fileAbsolutePath);
}
}
#endif
}
private IPythonModule LoadModuleFromDirectory(string searchPath, string moduleName)
{
Func <string, bool> isPackage = null;
if (!ModulePath.PythonVersionRequiresInitPyFiles(_langVersion))
{
isPackage = Directory.Exists;
}
ModulePath package;
try {
package = ModulePath.FromBasePathAndName(searchPath, moduleName, isPackage);
} catch (ArgumentException) {
return(null);
}
var db = EnsureSearchPathDB();
if (package.IsNativeExtension || package.IsCompiled)
{
db.LoadExtensionModule(package);
}
else
{
db.AddModule(package.FullName, AstPythonModule.FromFile(
this,
package.SourceFile,
_factory.GetLanguageVersion(),
package.FullName
));
}
if (db != _searchPathDb)
{
// Racing with the DB being invalidated.
// It's okay if we miss it here, so don't worry
// about taking the lock.
return(null);
}
var mod = db.GetModule(package.FullName);
if (!package.IsSpecialName)
{
int i = package.FullName.LastIndexOf('.');
if (i >= 1)
{
var parent = package.FullName.Remove(i);
var parentMod = db.GetModule(parent) as AstPythonModule;
if (parentMod != null)
{
parentMod.AddChildModule(package.Name, mod);
}
}
}
lock (_searchPathDbLock) {
if (db != _searchPathDb)
{
// Raced with the DB being invalidated
return(null);
}
}
return(mod);
}
internal void CopyFiles(string moduleName, bool @overrideSystemVersion)
{
var tempInstallationPath = _installationFileManager.GetTempInstallationPath(moduleName);
if (!Directory.Exists(tempInstallationPath.PhysicalPath))
{
throw new Exception("The temporary installation directory has been deleted.".Localize());
}
ModulePath modulePath = new ModulePath(moduleName);
Kooboo.IO.IOUtility.CopyDirectory(tempInstallationPath.PhysicalPath, modulePath.PhysicalPath);
var assemblyFiles = Directory.EnumerateFiles(modulePath.GetModuleInstallationFilePath("Bin").PhysicalPath);
var binPath = Settings.BinDirectory;
foreach (var item in assemblyFiles)
{
var fileName = Path.GetFileName(item);
var fileNameInBin = Path.Combine(binPath, fileName);
var exists = File.Exists(fileNameInBin);
if (!exists || (exists && overrideSystemVersion))
{
File.Copy(item, fileNameInBin, overrideSystemVersion);
}
_assemblyReferences.AddReference(fileNameInBin, moduleName);
}
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames)
{
var finding = new HashSet <string>(moduleNames);
var found = new HashSet <string>();
var withPackages = factory.PackageManager;
if (withPackages != null)
{
foreach (var m in finding)
{
if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid)
{
found.Add(m);
}
}
finding.ExceptWith(found);
if (!finding.Any())
{
// Found all of them, so stop searching
return(found);
}
}
var withDb = factory as PythonInterpreterFactoryWithDatabase;
if (withDb != null && withDb.IsCurrent)
{
var db = withDb.GetCurrentDatabase();
found.UnionWith(finding.Where(m => db.GetModule(m) != null));
// Always stop searching after this step
return(found);
}
if (withDb != null)
{
try {
var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb);
found.UnionWith(PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths)
.SelectMany()
.Select(g => g.ModuleName)
.Where(m => finding.Contains(m)));
} catch (InvalidOperationException) {
}
finding.ExceptWith(found);
if (!finding.Any())
{
// Found all of them, so stop searching
return(found);
}
}
return(await Task.Run(() => {
foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration))
{
if (finding.Remove(mp.ModuleName))
{
found.Add(mp.ModuleName);
}
if (!finding.Any())
{
break;
}
}
return found;
}));
}
