private IEnumerable<string> GetAssemblyFiles(string moduleName) { ModulePath modulePath = new ModulePath(moduleName); ModuleItemPath moduleBinPath = new ModuleItemPath(moduleName, "Bin"); if (Directory.Exists(moduleBinPath.PhysicalPath)) { return Directory.EnumerateFiles(moduleBinPath.PhysicalPath); } else { return new string[0]; } }
// Avoid hitting the filesystem, but exclude non-importable // paths. Ideally, we'd stop at the first path that's a known // search path, except we don't know search paths here. private static bool IsPackageCheck(string path) { return(ModulePath.IsImportable(PathUtils.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)))); }
private static string GetCoveragePath(IEnumerable <TestCase> tests) { string bestFile = null, bestClass = null, bestMethod = null; // Try and generate a friendly name for the coverage report. We use // the filename, class, and method. We include each one if we're // running from a single filename/class/method. When we have multiple // we drop the identifying names. If we have multiple files we // go to the top level directory... If all else fails we do "pycov". foreach (var test in tests) { string testFile, testClass, testMethod; TestReader.ParseFullyQualifiedTestName( test.FullyQualifiedName, out testFile, out testClass, out testMethod ); bestFile = UpdateBestFile(bestFile, test.CodeFilePath); if (bestFile != test.CodeFilePath) { // Different files, don't include class/methods even // if they happen to be the same. bestClass = bestMethod = ""; } bestClass = UpdateBest(bestClass, testClass); bestMethod = UpdateBest(bestMethod, testMethod); } string filename = ""; if (!String.IsNullOrWhiteSpace(bestFile)) { if (ModulePath.IsPythonSourceFile(bestFile)) { filename = ModulePath.FromFullPath(bestFile).ModuleName; } else { filename = Path.GetFileName(bestFile); } } else { filename = "pycov"; } if (!String.IsNullOrWhiteSpace(bestClass)) { filename += "_" + bestClass; } if (!String.IsNullOrWhiteSpace(bestMethod)) { filename += "_" + bestMethod; } filename += "_" + DateTime.Now.ToString("s").Replace(':', '_'); return(Path.Combine(Path.GetTempPath(), filename)); }
internal PythonAnalyzer AnalyzeDir(string dir, PythonLanguageVersion version = PythonLanguageVersion.V27, IEnumerable <string> excludeDirectories = null, CancellationToken?cancel = null) { List <string> files = new List <string>(); try { ISet <string> excluded = null; if (excludeDirectories != null) { excluded = new HashSet <string>(excludeDirectories, StringComparer.InvariantCultureIgnoreCase); } CollectFiles(dir, files, excluded); } catch (DirectoryNotFoundException) { return(null); } List <FileStreamReader> sourceUnits = new List <FileStreamReader>(); foreach (string file in files) { sourceUnits.Add( new FileStreamReader(file) ); } Stopwatch sw = new Stopwatch(); sw.Start(); long start0 = sw.ElapsedMilliseconds; // Explicitly specify the builtins name because we are using a 2.7 // interpreter for all versions. var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion()); var projectState = new PythonAnalyzer(fact, fact.CreateInterpreter(), "__builtin__"); projectState.ReloadModulesAsync().WaitAndUnwrapExceptions(); projectState.Limits = AnalysisLimits.GetStandardLibraryLimits(); var modules = new List <IPythonProjectEntry>(); foreach (var sourceUnit in sourceUnits) { try { modules.Add(projectState.AddModule( ModulePath.FromFullPath(sourceUnit.Path).ModuleName, sourceUnit.Path, null )); } catch (ArgumentException) { // Invalid module name, so skip the module } } long start1 = sw.ElapsedMilliseconds; Trace.TraceInformation("AddSourceUnit: {0} ms", start1 - start0); var nodes = new List <Microsoft.PythonTools.Parsing.Ast.PythonAst>(); for (int i = 0; i < modules.Count; i++) { PythonAst ast = null; try { var sourceUnit = sourceUnits[i]; ast = Parser.CreateParser(sourceUnit, version).ParseFile(); } catch (Exception) { } nodes.Add(ast); } long start2 = sw.ElapsedMilliseconds; Trace.TraceInformation("Parse: {0} ms", start2 - start1); for (int i = 0; i < modules.Count; i++) { var ast = nodes[i]; if (ast != null) { modules[i].UpdateTree(ast, null); } } long start3 = sw.ElapsedMilliseconds; for (int i = 0; i < modules.Count; i++) { Trace.TraceInformation("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path); var ast = nodes[i]; if (ast != null) { modules[i].Analyze(cancel ?? CancellationToken.None, true); } } if (modules.Count > 0) { Trace.TraceInformation("Analyzing queue"); modules[0].AnalysisGroup.AnalyzeQueuedEntries(cancel ?? CancellationToken.None); } long start4 = sw.ElapsedMilliseconds; Trace.TraceInformation("Analyze: {0} ms", start4 - start3); return(projectState); }
/// <summary> /// For Saving Module After Decrypting /// </summary> public void Save() { if (Module.IsILOnly) { var Options = new ModuleWriterOptions(Module) { Logger = DummyLogger.NoThrowInstance }; var NewPath = Module.Kind.Equals(ModuleKind.Dll) ? ModulePath.Replace(".dll", "-Decrypted.dll") : ModulePath.Replace(".exe", "-Decrypted.exe"); Module.Write(NewPath, Options); Log.Information($"Module Saved : {NewPath}"); System.Console.ReadKey(); } else { var Options = new NativeModuleWriterOptions(Module, false) { Logger = DummyLogger.NoThrowInstance }; var NewPath = Module.Kind.Equals(ModuleKind.Dll) ? ModulePath.Replace(".dll", "-Decrypted.dll") : ModulePath.Replace(".exe", "-Decrypted.exe"); Module.NativeWrite(NewPath, Options); Log.Information($"Module Saved : {NewPath}"); System.Console.ReadKey(); } }
private Task <IProjectEntry> AddFileAsync(Uri documentUri, Uri fromSearchPath, IAnalysisCookie cookie = null) { var item = _projectFiles.GetEntry(documentUri, throwIfMissing: false); if (item != null) { return(Task.FromResult(item)); } IEnumerable <string> aliases = null; var path = GetLocalPath(documentUri); if (fromSearchPath != null) { if (ModulePath.FromBasePathAndFile_NoThrow(GetLocalPath(fromSearchPath), path, out var mp)) { aliases = new[] { mp.ModuleName }; } } else { aliases = GetImportNames(documentUri).Select(mp => mp.ModuleName).ToArray(); } if (!(aliases?.Any() ?? false)) { aliases = new[] { Path.GetFileNameWithoutExtension(path) }; } var reanalyzeEntries = aliases.SelectMany(a => _analyzer.GetEntriesThatImportModule(a, true)).ToArray(); var first = aliases.FirstOrDefault(); var pyItem = _analyzer.AddModule(first, path, documentUri, cookie); item = pyItem; foreach (var a in aliases.Skip(1)) { _analyzer.AddModuleAlias(first, a); } var actualItem = _projectFiles.GetOrAddEntry(documentUri, item); if (actualItem != item) { return(Task.FromResult(actualItem)); } if (_clientCaps?.python?.analysisUpdates ?? false) { pyItem.OnNewAnalysis += ProjectEntry_OnNewAnalysis; } if (item is IDocument doc) { EnqueueItem(doc); } if (reanalyzeEntries != null) { foreach (var entryRef in reanalyzeEntries) { _queue.Enqueue(entryRef, AnalysisPriority.Low); } } return(Task.FromResult(item)); }
/// <summary> /// Determines whether the specified directory is an importable package. /// </summary> public bool IsPackage(string directory) { return(ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ? !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : Directory.Exists(directory)); }
private void AstNativeBuiltinScrape(PythonVersion version) { AstScrapedPythonModule.KeepAst = true; version.AssertInstalled(); using (var analysis = CreateAnalysis(version)) { try { var fact = (AstPythonInterpreterFactory)analysis.Analyzer.InterpreterFactory; var interp = (AstPythonInterpreter)analysis.Analyzer.Interpreter; var ctxt = interp.CreateModuleContext(); var dllsDir = PathUtils.GetAbsoluteDirectoryPath(fact.Configuration.PrefixPath, "DLLs"); if (!Directory.Exists(dllsDir)) { Assert.Inconclusive("Configuration does not have DLLs"); } var report = new List<string>(); var permittedImports = fact.GetLanguageVersion().Is2x() ? new[] { interp.BuiltinModuleName, "exceptions" } : new[] { interp.BuiltinModuleName }; foreach (var pyd in PathUtils.EnumerateFiles(dllsDir, "*", recurse: false).Where(ModulePath.IsPythonFile)) { var mp = ModulePath.FromFullPath(pyd); if (mp.IsDebug) { continue; } Console.WriteLine("Importing {0} from {1}", mp.ModuleName, mp.SourceFile); var mod = interp.ImportModule(mp.ModuleName); Assert.IsInstanceOfType(mod, typeof(AstScrapedPythonModule)); mod.Imported(ctxt); var modPath = fact.GetCacheFilePath(pyd); Assert.IsTrue(File.Exists(modPath), "No cache file created"); _moduleCache = File.ReadAllText(modPath); var errors = ((AstScrapedPythonModule)mod).ParseErrors ?? Enumerable.Empty<string>(); foreach (var err in errors) { Console.WriteLine(err); } Assert.AreEqual(0, errors.Count(), "Parse errors occurred"); var ast = ((AstScrapedPythonModule)mod).Ast; var imports = ((Ast.SuiteStatement)ast.Body).Statements .OfType<Ast.ImportStatement>() .SelectMany(s => s.Names) .Select(n => n.MakeString()) .Except(permittedImports) .ToArray(); // We expect no imports (after excluding builtins) report.AddRange(imports.Select(n => $"{mp.ModuleName} imported {n}")); _moduleCache = null; } AssertUtil.ContainsExactly(report); } finally { _analysisLog = analysis.GetLogContent(CultureInfo.InvariantCulture); } } }
private async Task <IProjectEntry> AddFileAsync(Uri documentUri, Uri fromSearchPath, IAnalysisCookie cookie = null) { var item = ProjectFiles.GetEntry(documentUri, throwIfMissing: false); if (item != null) { return(item); } string[] aliases = null; var path = GetLocalPath(documentUri); if (fromSearchPath != null) { if (ModulePath.FromBasePathAndFile_NoThrow(GetLocalPath(fromSearchPath), path, out var mp)) { aliases = new[] { mp.ModuleName }; } } else { aliases = GetImportNames(documentUri).Select(mp => mp.ModuleName).ToArray(); } if (aliases.IsNullOrEmpty()) { aliases = new[] { Path.GetFileNameWithoutExtension(path) }; } var reanalyzeEntries = aliases.SelectMany(a => Analyzer.GetEntriesThatImportModule(a, true)).ToArray(); var first = aliases.FirstOrDefault(); var pyItem = Analyzer.AddModule(first, path, documentUri, cookie); item = pyItem; foreach (var a in aliases.Skip(1)) { Analyzer.AddModuleAlias(first, a); } var actualItem = ProjectFiles.GetOrAddEntry(documentUri, item); if (actualItem != item) { return(actualItem); } pyItem.OnNewAnalysis += (o, e) => OnPythonEntryNewAnalysis(pyItem); if (item is IDocument doc) { EnqueueItem(doc); } foreach (var entryRef in reanalyzeEntries) { _queue.Enqueue(entryRef, AnalysisPriority.Low); } return(item); }
public void Imported(IModuleContext context) { if (_scraped) { return; } _scraped = true; var interp = context as AstPythonInterpreter; var fact = interp?.Factory; if (fact == null || !File.Exists(fact.Configuration.InterpreterPath)) { return; } ModulePath mp = AstPythonInterpreterFactory.FindModule(fact, _filePath); if (string.IsNullOrEmpty(mp.FullName)) { return; } var sm = PythonToolsInstallPath.TryGetFile("scrape_module.py", GetType().Assembly); if (!File.Exists(sm)) { return; } Stream code = null; using (var p = ProcessOutput.RunHiddenAndCapture( fact.Configuration.InterpreterPath, "-E", sm, mp.LibraryPath, mp.ModuleName )) { p.Wait(); if (p.ExitCode == 0) { var ms = new MemoryStream(); code = ms; using (var sw = new StreamWriter(ms, Encoding.UTF8, 4096, true)) { foreach (var line in p.StandardOutputLines) { sw.WriteLine(line); } } } } if (code == null) { return; } PythonAst ast; code.Seek(0, SeekOrigin.Begin); using (var sr = new StreamReader(code, Encoding.UTF8)) using (var parser = Parser.CreateParser(sr, fact.GetLanguageVersion())) { ast = parser.ParseFile(); } lock (_members) { var walker = new AstAnalysisWalker(interp, ast, this, _filePath, _members, false); ast.Walk(walker); } }
// Avoid hitting the filesystem, but exclude non-importable // paths. Ideally, we'd stop at the first path that's a known // search path, except we don't know search paths here. private static bool IsPackageCheck(string path) { return(ModulePath.IsImportable(PathUtils.GetFileOrDirectoryName(path))); }
private void ConectionReceivedEvent(object sender, EventReceivedEventArgs e) { switch (e.Name) { case TP.ResultEvent.Name: var result = (TP.ResultEvent)e.Event; TestOutcome outcome = TestOutcome.None; switch (result.outcome) { case "passed": outcome = TestOutcome.Passed; break; case "failed": outcome = TestOutcome.Failed; break; case "skipped": outcome = TestOutcome.Skipped; break; } var testResult = new TestResult(_curTest); RecordEnd( _frameworkHandle, _curTest, testResult, _stdOut.ToString(), _stdErr.ToString(), outcome, result ); _stdOut.Clear(); _stdErr.Clear(); break; case TP.StartEvent.Name: var start = (TP.StartEvent)e.Event; _curTest = null; foreach (var test in _tests) { string testFile, testClass, testMethod; TestReader.ParseFullyQualifiedTestName( test.FullyQualifiedName, out testFile, out testClass, out testMethod ); string testFilePath = PathUtils.GetAbsoluteFilePath(_settings.ProjectHome, test.CodeFilePath); var modulePath = ModulePath.FromFullPath(testFilePath); if (start.test == modulePath.ModuleName + "." + testClass + "." + testMethod) { _curTest = test; break; } } if (_curTest != null) { _frameworkHandle.RecordStart(_curTest); } else { Warning(Strings.Test_UnexpectedResult.FormatUI(start.classname, start.method)); } break; case TP.StdErrEvent.Name: var err = (TP.StdErrEvent)e.Event; _stdErr.Append(err.content); break; case TP.StdOutEvent.Name: var outp = (TP.StdOutEvent)e.Event; _stdOut.Append(outp.content); break; case TP.DoneEvent.Name: _done.Set(); break; } }
internal void RemoveModuleArea(string moduleName) { ModulePath modulePath = new ModulePath(moduleName); Kooboo.IO.IOUtility.DeleteDirectory(modulePath.PhysicalPath, true); }
/// <summary> /// Determines whether the specified directory is an importable package. /// </summary> private bool IsPackage(string directory) => ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ? !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : _fs.DirectoryExists(directory);
private async Task FullStdLibTest(InterpreterConfiguration configuration, params string[] skipModules) { configuration.AssertInstalled(); var moduleUri = TestData.GetDefaultModuleUri(); var moduleDirectory = Path.GetDirectoryName(moduleUri.LocalPath); var services = await CreateServicesAsync(moduleDirectory, configuration); var interpreter = services.GetService <IPythonInterpreter>(); var modules = ModulePath.GetModulesInLib(configuration.LibraryPath, configuration.SitePackagesPath).ToList(); var skip = new HashSet <string>(skipModules); skip.UnionWith(new[] { @"matplotlib.backends._backend_gdk", @"matplotlib.backends._backend_gtkagg", @"matplotlib.backends._gtkagg", "test.test_pep3131", "test.test_unicode_identifiers", "test.test_super" // nonlocal syntax error }); skip.UnionWith(modules.Select(m => m.FullName) .Where(n => n.StartsWith(@"test.badsyntax") || n.StartsWith("test.bad_coding"))); var anySuccess = false; var anyExtensionSuccess = false; var anyExtensionSeen = false; var anyParseError = false; foreach (var m in skip) { ((MainModuleResolution)interpreter.ModuleResolution).AddUnimportableModule(m); } var set = modules .Where(m => !skip.Contains(m.ModuleName)) .GroupBy(m => { var i = m.FullName.IndexOf('.'); return(i <= 0 ? m.FullName : m.FullName.Remove(i)); }) .SelectMany(g => g.Select(m => Tuple.Create(m, m.ModuleName))) .ToArray(); set = set.Where(x => x.Item2 != null && x.Item2.Contains("grammar")).ToArray(); var sb = new StringBuilder(); foreach (var r in set) { var module = interpreter.ModuleResolution.GetOrLoadModule(r.Item2); if (module != null) { sb.AppendLine($"import {module.Name}"); } } await GetAnalysisAsync(sb.ToString(), services); foreach (var r in set) { var modName = r.Item1; var mod = interpreter.ModuleResolution.GetOrLoadModule(r.Item2); anyExtensionSeen |= modName.IsNativeExtension; switch (mod) { case null: Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile); break; case CompiledPythonModule compiledPythonModule: var errors = compiledPythonModule.GetParseErrors().ToArray(); if (errors.Any()) { anyParseError = true; Trace.TraceError("Parse errors in {0}", modName.SourceFile); foreach (var e in errors) { Trace.TraceError(e.Message); } } else { anySuccess = true; anyExtensionSuccess |= modName.IsNativeExtension; } break; case IPythonModule _: { var filteredErrors = ((IDocument)mod).GetParseErrors().Where(e => !e.Message.Contains("encoding problem")).ToArray(); if (filteredErrors.Any()) { // Do not fail due to errors in installed packages if (!mod.FilePath.Contains("site-packages")) { anyParseError = true; } Trace.TraceError("Parse errors in {0}", modName.SourceFile); foreach (var e in filteredErrors) { Trace.TraceError(e.Message); } } else { anySuccess = true; anyExtensionSuccess |= modName.IsNativeExtension; } break; } } } Assert.IsTrue(anySuccess, "failed to import any modules at all"); Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules"); Assert.IsFalse(anyParseError, "parse errors occurred"); }
public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, ModulePath moduleName, CancellationToken cancel) { _typeDb = typeDb; _factory = factory ?? throw new ArgumentNullException(nameof(factory)); _moduleName = moduleName; _cancel = cancel; }
private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configuration) { configuration.AssertInstalled(); var moduleUri = TestData.GetDefaultModuleUri(); var moduleDirectory = Path.GetDirectoryName(moduleUri.LocalPath); var services = await CreateServicesAsync(moduleDirectory, configuration); var interpreter = services.GetService <IPythonInterpreter>(); // TODO: this is Windows only var dllsDir = Path.Combine(Path.GetDirectoryName(interpreter.Configuration.LibraryPath), "DLLs"); if (!Directory.Exists(dllsDir)) { Assert.Inconclusive("Configuration does not have DLLs"); } var report = new List <string>(); var permittedImports = interpreter.LanguageVersion.Is2x() ? new[] { interpreter.ModuleResolution.BuiltinModuleName, "exceptions" } : new[] { interpreter.ModuleResolution.BuiltinModuleName }; var modules = new List <IPythonModule>(); foreach (var pyd in PathUtils.EnumerateFiles(dllsDir, "*", recurse: false).Where(ModulePath.IsPythonFile)) { var mp = ModulePath.FromFullPath(pyd); if (mp.IsDebug) { continue; } Console.WriteLine(@"Importing {0} from {1}", mp.ModuleName, mp.SourceFile); modules.Add(interpreter.ModuleResolution.GetOrLoadModule(mp.ModuleName)); } foreach (var mod in modules) { Assert.IsInstanceOfType(mod, typeof(CompiledPythonModule)); await((ModuleCache)interpreter.ModuleResolution.ModuleCache).CacheWritingTask; var modPath = interpreter.ModuleResolution.ModuleCache.GetCacheFilePath(mod.FilePath); Assert.IsTrue(File.Exists(modPath), "No cache file created"); var doc = (IDocument)mod; var ast = await doc.GetAstAsync(); var errors = doc.GetParseErrors().ToArray(); foreach (var err in errors) { Console.WriteLine(err); } Assert.AreEqual(0, errors.Length, "Parse errors occurred"); var imports = ((SuiteStatement)ast.Body).Statements .OfType <ImportStatement>() .SelectMany(s => s.Names) .Select(n => n.MakeString()) .Except(permittedImports) .ToArray(); // We expect no imports (after excluding builtins) report.AddRange(imports.Select(n => $"{mod.Name} imported {n}")); } report.Should().BeEmpty(); }
public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, ModulePath moduleName, CancellationToken cancel) { _typeDb = typeDb; _factory = factory; _moduleName = moduleName; _cancel = cancel; }
private async Task FullStdLibTest(PythonVersion v, params string[] skipModules) { v.AssertInstalled(); var factory = new AstPythonInterpreterFactory(v.Configuration, new InterpreterFactoryCreationOptions { DatabasePath = TestData.GetTempPath(), UseExistingCache = false }); var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList(); var skip = new HashSet<string>(skipModules); skip.UnionWith(new[] { "matplotlib.backends._backend_gdk", "matplotlib.backends._backend_gtkagg", "matplotlib.backends._gtkagg", }); bool anySuccess = false; bool anyExtensionSuccess = false, anyExtensionSeen = false; bool anyParseError = false; using (var analyzer = new PythonAnalysis(factory)) { try { var tasks = new List<Task<Tuple<ModulePath, IPythonModule>>>(); var interp = (AstPythonInterpreter)analyzer.Analyzer.Interpreter; foreach (var m in skip) { interp.AddUnimportableModule(m); } foreach (var r in modules .Where(m => !skip.Contains(m.ModuleName)) .GroupBy(m => { int i = m.FullName.IndexOf('.'); return i <= 0 ? m.FullName : m.FullName.Remove(i); }) .AsParallel() .SelectMany(g => g.Select(m => Tuple.Create(m, interp.ImportModule(m.ModuleName)))) ) { var modName = r.Item1; var mod = r.Item2; anyExtensionSeen |= modName.IsNativeExtension; if (mod == null) { Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile); } else if (mod is AstScrapedPythonModule smod) { if (smod.ParseErrors?.Any() ?? false) { anyParseError = true; Trace.TraceError("Parse errors in {0}", modName.SourceFile); foreach (var e in smod.ParseErrors) { Trace.TraceError(e); } } else { anySuccess = true; anyExtensionSuccess |= modName.IsNativeExtension; mod.GetMemberNames(analyzer.ModuleContext).ToList(); } } else if (mod is AstPythonModule) { // pass } else { Trace.TraceError("imported {0} as type {1}", modName.ModuleName, mod.GetType().FullName); } } } finally { _analysisLog = analyzer.GetLogContent(CultureInfo.InvariantCulture); } } Assert.IsTrue(anySuccess, "failed to import any modules at all"); Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules"); Assert.IsFalse(anyParseError, "parse errors occurred"); }
private string GenerateDbFile(IPythonInterpreterFactory interpreter, ModulePath moduleName, List <string> existingModules, string dbFile, FileStream fs) { // we need to generate the DB file dbFile = Path.Combine(ReferencesDatabasePath, moduleName + ".$project.idb"); int retryCount = 0; while (File.Exists(dbFile)) { dbFile = Path.Combine(ReferencesDatabasePath, moduleName + "." + ++retryCount + ".$project.idb"); } var args = new List <string> { PythonToolsInstallPath.GetFile("ExtensionScraper.py"), "scrape", }; if (moduleName.IsNativeExtension) { args.Add("-"); args.Add(moduleName.SourceFile); } else { args.Add(moduleName.ModuleName); args.Add(moduleName.LibraryPath); } args.Add(Path.ChangeExtension(dbFile, null)); using (var output = interpreter.Run(args.ToArray())) { if (_cancel.CanBeCanceled) { if (WaitHandle.WaitAny(new[] { _cancel.WaitHandle, output.WaitHandle }) != 1) { // we were cancelled return(null); } } else { output.Wait(); } if (output.ExitCode == 0) { // [FileName]|interpGuid|interpVersion|DateTimeStamp|[db_file.idb] // save the new entry in the DB file existingModules.Add( String.Format("{0}|{1}|{2}|{3}", moduleName.SourceFile, interpreter.Configuration.Id, new FileInfo(moduleName.SourceFile).LastWriteTime.ToString("O"), dbFile ) ); fs.Seek(0, SeekOrigin.Begin); fs.SetLength(0); using (var sw = new StreamWriter(fs)) { sw.Write(String.Join(Environment.NewLine, existingModules)); sw.Flush(); } } else { throw new CannotAnalyzeExtensionException(string.Join(Environment.NewLine, output.StandardErrorLines)); } } return(dbFile); }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { ValidateArg.NotNull(sources, "sources"); ValidateArg.NotNull(discoverySink, "discoverySink"); var buildEngine = new MSBuild.ProjectCollection(); try { // Load all the test containers passed in (.pyproj msbuild files) foreach (string source in sources) { buildEngine.LoadProject(source); } foreach (var proj in buildEngine.LoadedProjects) { using (var provider = new MSBuildProjectInterpreterFactoryProvider(_interpreterService, proj)) { try { provider.DiscoverInterpreters(); } catch (InvalidDataException) { // This exception can be safely ignored here. } var factory = provider.ActiveInterpreter; if (factory == _interpreterService.NoInterpretersValue) { if (logger != null) { logger.SendMessage(TestMessageLevel.Warning, "No interpreters available for project " + proj.FullPath); } continue; } var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, proj.GetPropertyValue(PythonConstants.ProjectHomeSetting) ?? ".")); // Do the analysis even if the database is not up to date. At // worst, we'll get no results. using (var analyzer = new TestAnalyzer( factory, proj.FullPath, projectHome, TestExecutor.ExecutorUri )) { // Provide all files to the test analyzer foreach (var item in proj.GetItems("Compile")) { string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude); string fullName; try { fullName = ModulePath.FromFullPath(fileAbsolutePath).ModuleName; } catch (ArgumentException) { if (logger != null) { logger.SendMessage(TestMessageLevel.Warning, "File has an invalid module name: " + fileAbsolutePath); } continue; } try { using (var reader = new StreamReader(fileAbsolutePath)) { analyzer.AddModule(fullName, fileAbsolutePath, reader); } } catch (FileNotFoundException) { // user deleted file, we send the test update, but the project // isn't saved. #if DEBUG } catch (Exception ex) { if (logger != null) { logger.SendMessage(TestMessageLevel.Warning, "Failed to discover tests in " + fileAbsolutePath); logger.SendMessage(TestMessageLevel.Informational, ex.ToString()); } } #else } catch (Exception) { if (logger != null) { logger.SendMessage(TestMessageLevel.Warning, "Failed to discover tests in " + fileAbsolutePath); } } #endif }
private IPythonModule LoadModuleFromDirectory(string searchPath, string moduleName) { Func <string, bool> isPackage = null; if (!ModulePath.PythonVersionRequiresInitPyFiles(_langVersion)) { isPackage = Directory.Exists; } ModulePath package; try { package = ModulePath.FromBasePathAndName(searchPath, moduleName, isPackage); } catch (ArgumentException) { return(null); } var db = EnsureSearchPathDB(); if (package.IsNativeExtension || package.IsCompiled) { db.LoadExtensionModule(package); } else { db.AddModule(package.FullName, AstPythonModule.FromFile( this, package.SourceFile, _factory.GetLanguageVersion(), package.FullName )); } if (db != _searchPathDb) { // Racing with the DB being invalidated. // It's okay if we miss it here, so don't worry // about taking the lock. return(null); } var mod = db.GetModule(package.FullName); if (!package.IsSpecialName) { int i = package.FullName.LastIndexOf('.'); if (i >= 1) { var parent = package.FullName.Remove(i); var parentMod = db.GetModule(parent) as AstPythonModule; if (parentMod != null) { parentMod.AddChildModule(package.Name, mod); } } } lock (_searchPathDbLock) { if (db != _searchPathDb) { // Raced with the DB being invalidated return(null); } } return(mod); }
internal void CopyFiles(string moduleName, bool @overrideSystemVersion) { var tempInstallationPath = _installationFileManager.GetTempInstallationPath(moduleName); if (!Directory.Exists(tempInstallationPath.PhysicalPath)) { throw new Exception("The temporary installation directory has been deleted.".Localize()); } ModulePath modulePath = new ModulePath(moduleName); Kooboo.IO.IOUtility.CopyDirectory(tempInstallationPath.PhysicalPath, modulePath.PhysicalPath); var assemblyFiles = Directory.EnumerateFiles(modulePath.GetModuleInstallationFilePath("Bin").PhysicalPath); var binPath = Settings.BinDirectory; foreach (var item in assemblyFiles) { var fileName = Path.GetFileName(item); var fileNameInBin = Path.Combine(binPath, fileName); var exists = File.Exists(fileNameInBin); if (!exists || (exists && overrideSystemVersion)) { File.Copy(item, fileNameInBin, overrideSystemVersion); } _assemblyReferences.AddReference(fileNameInBin, moduleName); } }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames) { var finding = new HashSet <string>(moduleNames); var found = new HashSet <string>(); var withPackages = factory.PackageManager; if (withPackages != null) { foreach (var m in finding) { if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid) { found.Add(m); } } finding.ExceptWith(found); if (!finding.Any()) { // Found all of them, so stop searching return(found); } } var withDb = factory as PythonInterpreterFactoryWithDatabase; if (withDb != null && withDb.IsCurrent) { var db = withDb.GetCurrentDatabase(); found.UnionWith(finding.Where(m => db.GetModule(m) != null)); // Always stop searching after this step return(found); } if (withDb != null) { try { var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb); found.UnionWith(PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths) .SelectMany() .Select(g => g.ModuleName) .Where(m => finding.Contains(m))); } catch (InvalidOperationException) { } finding.ExceptWith(found); if (!finding.Any()) { // Found all of them, so stop searching return(found); } } return(await Task.Run(() => { foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration)) { if (finding.Remove(mp.ModuleName)) { found.Add(mp.ModuleName); } if (!finding.Any()) { break; } } return found; })); }