private static PythonVersion GetIronPythonVersion(bool x64) {
var exeName = x64 ? "ipy64.exe" : "ipy.exe";
using (var ipy = Registry.LocalMachine.OpenSubKey("SOFTWARE\\IronPython")) {
if (ipy != null) {
using (var twoSeven = ipy.OpenSubKey("2.7")) {
if (twoSeven != null) {
var installPath = twoSeven.OpenSubKey("InstallPath");
if (installPath != null) {
var res = installPath.GetValue("") as string;
if (res != null) {
return new PythonVersion(
Path.Combine(res, exeName),
PythonLanguageVersion.V27,
x64 ? IronPython64Guid : IronPythonGuid
);
}
}
}
}
}
}
var ver = new PythonVersion("C:\\Program Files (x86)\\IronPython 2.7\\" + exeName, PythonLanguageVersion.V27, IronPythonGuid);
if (File.Exists(ver.InterpreterPath)) {
return ver;
}
return null;
}
private static PythonVersion GetIronPythonVersion(bool x64)
{
var exeName = x64 ? "ipy64.exe" : "ipy.exe";
using (var ipy = Registry.LocalMachine.OpenSubKey("SOFTWARE\\IronPython")) {
if (ipy != null)
{
using (var twoSeven = ipy.OpenSubKey("2.7")) {
if (twoSeven != null)
{
var installPath = twoSeven.OpenSubKey("InstallPath");
if (installPath != null)
{
var res = installPath.GetValue("") as string;
if (res != null)
{
return(new PythonVersion(Path.Combine(res, exeName), PythonLanguageVersion.V27, IronPythonGuid));
}
}
}
}
}
}
var ver = new PythonVersion("C:\\Program Files (x86)\\IronPython 2.7\\" + exeName, PythonLanguageVersion.V27, IronPythonGuid);
if (File.Exists(ver.Path))
{
return(ver);
}
return(null);
}
public static void AssertInstalled(this PythonVersion pyVersion, string customMessage)
{
if (pyVersion == null || !File.Exists(pyVersion.InterpreterPath))
{
Assert.Inconclusive(customMessage);
}
}
public static void AssertInstalled(this PythonVersion self)
{
if (self == null || !File.Exists(self.InterpreterPath))
{
Assert.Inconclusive("Python interpreter not installed");
}
}
private static PythonVersion GetIronPythonVersion(bool x64)
{
var exeName = x64 ? "ipy64.exe" : "ipy.exe";
using (var ipy = Registry.LocalMachine.OpenSubKey("SOFTWARE\\IronPython")) {
if (ipy != null)
{
using (var twoSeven = ipy.OpenSubKey("2.7")) {
if (twoSeven != null)
{
var installPath = twoSeven.OpenSubKey("InstallPath");
if (installPath != null)
{
var res = installPath.GetValue("") as string;
if (res != null)
{
// IronPython changed to Any CPU for ipy.exe and ipy32.exe for 32-bit in 2.7.8
if (File.Exists(Path.Combine(res, "ipy32.exe")))
{
exeName = x64 ? "ipy.exe" : "ipy32.exe";
}
return(new PythonVersion(
new InterpreterConfiguration(
x64 ? "IronPython|2.7-64" : "IronPython|2.7-32",
string.Format("IronPython {0} 2.7", x64 ? "64-bit" : "32-bit"),
res,
Path.Combine(res, exeName),
arch: x64?InterpreterArchitecture.x64: InterpreterArchitecture.x86,
version: new Version(2, 7),
pathVar: "IRONPYTHONPATH"
),
ironPython: true
));
}
}
}
}
}
}
var ver = new PythonVersion(new InterpreterConfiguration(
"IronPython|2.7-32",
"IronPython 32-bit 2.7",
"C:\\Program Files (x86)\\IronPython 2.7\\",
"C:\\Program Files (x86)\\IronPython 2.7\\" + exeName,
arch: InterpreterArchitecture.x86,
version: new Version(2, 7),
pathVar: "IRONPYTHONPATH"
), ironPython: true);
if (File.Exists(ver.InterpreterPath))
{
return(ver);
}
return(null);
}
static DefaultInterpreterSetter Init(PythonVisualStudioApp app, PythonVersion interp, bool install) {
var dis = app.SelectDefaultInterpreter(interp);
try {
if (install) {
dis.CurrentDefault.PipInstall("-U virtualenv");
}
var r = dis;
dis = null;
return r;
} finally {
dis?.Dispose();
}
}
/// <summary>
/// Creates a Python virtual environment in specified directory.
/// </summary>
public static void CreateVirtualEnv(this PythonVersion pyVersion, string envPath)
{
var virtualEnvModule = (pyVersion.Version < PythonLanguageVersion.V30) ? "virtualenv" : "venv";
using (var p = ProcessOutput.RunHiddenAndCapture(pyVersion.InterpreterPath, "-m", virtualEnvModule, envPath)) {
Console.WriteLine(p.Arguments);
Assert.IsTrue(p.Wait(TimeSpan.FromMinutes(3)));
Console.WriteLine(string.Join(Environment.NewLine, p.StandardOutputLines.Concat(p.StandardErrorLines)));
Assert.AreEqual(0, p.ExitCode);
}
Assert.IsTrue(File.Exists(Path.Combine(envPath, "scripts", "python.exe")));
}
public static void AssertInstalled(this PythonVersion pyVersion)
{
if (pyVersion == null || !File.Exists(pyVersion.InterpreterPath))
{
if (pyVersion == null)
{
Assert.Inconclusive("Python interpreter is not installed. pyVersion is null. ");
}
else
{
Assert.Inconclusive(string.Format("Python version {0} is not installed.", pyVersion.Configuration.Version.ToString()));
}
}
}
/// <summary>
/// Creates a Python virtual environment in specified directory and installs the specified packages.
/// </summary>
public static string CreateVirtualEnv(this PythonVersion pyVersion, VirtualEnvName envName, IEnumerable <string> packages, string rootDirectory = null)
{
var envPath = pyVersion.CreateVirtualEnv(envName, rootDirectory);
var envPythonExePath = Path.Combine(envPath, "scripts", "python.exe");
foreach (var package in packages.MaybeEnumerate())
{
using (var output = ProcessOutput.RunHiddenAndCapture(envPythonExePath, "-m", "pip", "install", package)) {
Assert.IsTrue(output.Wait(TimeSpan.FromSeconds(30)));
Assert.AreEqual(0, output.ExitCode);
}
}
return(envPath);
}
/// <summary>
/// Creates a Python virtual environment in vEnvPath directory and installs the specified packages
/// </summary>
/// <param name="pyVersion"></param>
/// <param name="virtualEnvPath"></param>
/// <param name="packages"></param>
public static void CreatePythonVirtualEnvWithPkgs(this PythonVersion pyVersion, string virtualEnvPath, string[] packages)
{
pyVersion.CreatePythonVirtualEnv(virtualEnvPath);
var envPythonExePath = Path.Combine(virtualEnvPath, "scripts", "python.exe");
foreach (var package in packages.MaybeEnumerate())
{
using (var output = ProcessOutput.RunHiddenAndCapture(envPythonExePath, "-m", "pip", "install", package)) {
Assert.IsTrue(output.Wait(TimeSpan.FromSeconds(30)));
Assert.AreEqual(0, output.ExitCode);
}
}
}
/// <summary>
/// Creates a Python virtual environment in specified directory.
/// </summary>
public static string CreateVirtualEnv(this PythonVersion pyVersion, VirtualEnvName envName, string rootDirectory = null)
{
// Only create this virtual env if it doesn't already exist.
var envPath = GetVirtualEnvPath(envName, rootDirectory);
if (!File.Exists(Path.Combine(envPath, "scripts", "python.exe")))
{
var virtualEnvModule = (pyVersion.Version < PythonLanguageVersion.V30) ? "virtualenv" : "venv";
using (var p = ProcessOutput.RunHiddenAndCapture(pyVersion.InterpreterPath, "-m", virtualEnvModule, envPath)) {
Console.WriteLine(p.Arguments);
Assert.IsTrue(p.Wait(TimeSpan.FromMinutes(3)));
Console.WriteLine(string.Join(Environment.NewLine, p.StandardOutputLines.Concat(p.StandardErrorLines)));
Assert.AreEqual(0, p.ExitCode);
}
}
Assert.IsTrue(File.Exists(Path.Combine(envPath, "scripts", "python.exe")));
return(envPath);
}
internal static PythonProcess DebugProcess(this PythonDebugger debugger, PythonVersion version, string filename, Action<PythonProcess, PythonThread> onLoaded = null, bool resumeOnProcessLoaded = true, string interpreterOptions = null, PythonDebugOptions debugOptions = PythonDebugOptions.RedirectOutput, string cwd = null, string arguments = "") {
string fullPath = Path.GetFullPath(filename);
string dir = cwd ?? Path.GetFullPath(Path.GetDirectoryName(filename));
if (!String.IsNullOrEmpty(arguments)) {
arguments = "\"" + fullPath + "\" " + arguments;
} else {
arguments = "\"" + fullPath + "\"";
}
var process = debugger.CreateProcess(version.Version, version.InterpreterPath, arguments, dir, "", interpreterOptions, debugOptions);
process.DebuggerOutput += (sender, args) => {
Console.WriteLine("{0}: {1}", args.Thread.Id, args.Output);
};
process.ProcessLoaded += (sender, args) => {
if (onLoaded != null) {
onLoaded(process, args.Thread);
}
if (resumeOnProcessLoaded) {
process.Resume();
}
};
return process;
}
private static PythonVersion GetIronPythonVersion(bool x64) {
var exeName = x64 ? "ipy64.exe" : "ipy.exe";
using (var ipy = Registry.LocalMachine.OpenSubKey("SOFTWARE\\IronPython")) {
if (ipy != null) {
using (var twoSeven = ipy.OpenSubKey("2.7")) {
if (twoSeven != null) {
var installPath = twoSeven.OpenSubKey("InstallPath");
if (installPath != null) {
var res = installPath.GetValue("") as string;
if (res != null) {
return new PythonVersion(
new InterpreterConfiguration(
x64 ? "IronPython|2.7-64" : "IronPython|2.7-32",
string.Format("IronPython {0} 2.7", x64 ? "64-bit" : "32-bit"),
res,
Path.Combine(res, exeName),
arch: x64 ? InterpreterArchitecture.x64 : InterpreterArchitecture.x86,
version: new Version(2, 7)
),
ironPython: true
);
}
}
}
}
}
}
var ver = new PythonVersion(new InterpreterConfiguration(
"IronPython|2.7-32",
"IronPython 32-bit 2.7",
"C:\\Program Files (x86)\\IronPython 2.7\\",
"C:\\Program Files (x86)\\IronPython 2.7\\" + exeName,
arch: InterpreterArchitecture.x86,
version: new Version(2, 7)
), ironPython: true);
if (File.Exists(ver.InterpreterPath)) {
return ver;
}
return null;
}
private void TestMutateStdLib(PythonVersion version) {
version.AssertInstalled();
for (int i = 0; i < 100; i++) {
int seed = (int)DateTime.Now.Ticks;
var random = new Random(seed);
Console.WriteLine("Seed == " + seed);
Console.WriteLine("Testing version {0} {1}", version.Version, Path.Combine(version.PrefixPath, "Lib"));
int ran = 0, succeeded = 0;
string[] files;
try {
files = Directory.GetFiles(Path.Combine(version.PrefixPath, "Lib"));
} catch (DirectoryNotFoundException) {
continue;
}
foreach (var file in files) {
try {
if (file.EndsWith(".py")) {
ran++;
TestOneFileMutated(file, version.Version, random);
succeeded++;
}
} catch (Exception e) {
Console.WriteLine(e);
Console.WriteLine("Failed: {0}", file);
break;
}
}
Assert.AreEqual(ran, succeeded);
}
}
private void BuiltinsProfile(PythonVersion interp, string[] expectedFunctions, string[] expectedNonFunctions) {
interp.AssertInstalled();
EnvDTE.Project project;
IPythonProfiling profiling;
using (var app = OpenProfileTestProject(out project, out profiling, null)) {
var session = LaunchProcess(app, profiling, interp.Id.ToString() + ";" + interp.Version.ToVersion().ToString(),
TestData.GetPath(@"TestData\ProfileTest\BuiltinsProfile.py"),
TestData.GetPath(@"TestData\ProfileTest"),
"",
false
);
try {
while (profiling.IsProfiling) {
Thread.Sleep(100);
}
var report = session.GetReport(1);
var filename = report.Filename;
Assert.IsTrue(filename.Contains("BuiltinsProfile"));
Assert.IsNull(session.GetReport(2));
Assert.IsNotNull(session.GetReport(report.Filename));
Assert.IsTrue(File.Exists(filename));
if (expectedFunctions != null && expectedFunctions.Length > 0) {
VerifyReport(report, true, expectedFunctions);
}
if (expectedNonFunctions != null && expectedNonFunctions.Length > 0) {
VerifyReport(report, false, expectedNonFunctions);
}
} finally {
profiling.RemoveSession(session, false);
}
}
}
private static string StdLibWorker(PythonVersion curVersion) {
var files = new List<string>();
CollectFiles(curVersion.LibPath, files, new[] { "site-packages" });
var skippedFiles = new HashSet<string>(new[] {
"py3_test_grammar.py", // included in 2x distributions but includes 3x grammar
"py2_test_grammar.py", // included in 3x distributions but includes 2x grammar
"proxy_base.py", // included in Qt port to Py3k but installed in 2.x distributions
"test_pep3131.py" // we need to update to support this.
});
var errorSink = new CollectingErrorSink();
var errors = new Dictionary<string, List<ErrorResult>>();
foreach (var file in files) {
string filename = Path.GetFileName(file);
if (skippedFiles.Contains(filename) || filename.StartsWith("badsyntax_") || filename.StartsWith("bad_coding") || file.IndexOf("\\lib2to3\\tests\\") != -1) {
continue;
}
using (var parser = Parser.CreateParser(new StreamReader(file), curVersion.Version, new ParserOptions() { ErrorSink = errorSink })) {
var ast = parser.ParseFile();
}
if (errorSink.Errors.Count != 0) {
var fileErrors = errorSink.Errors.ToList();
if (curVersion.Configuration.Version == new Version(3, 5)) {
// TODO: https://github.com/Microsoft/PTVS/issues/337
fileErrors.RemoveAll(e => {
return e.Message == "non-keyword arg after keyword arg";
});
}
if (fileErrors.Any()) {
errors["\"" + file + "\""] = fileErrors;
errorSink = new CollectingErrorSink();
}
}
}
if (errors.Count != 0) {
StringBuilder errorList = new StringBuilder();
foreach (var keyValue in errors) {
errorList.Append(keyValue.Key + " :" + Environment.NewLine);
foreach (var error in keyValue.Value) {
errorList.AppendFormat(" {0} {1}{2}", error.Span, error.Message, Environment.NewLine);
}
}
return errorList.ToString();
}
return null;
}
static DefaultInterpreterSetter Init(PythonVisualStudioApp app, PythonVersion interp, bool install) {
return app.SelectDefaultInterpreter(interp, install ? "virtualenv" : null);
}
private void TestOpen(PythonVersion path) {
path.AssertInstalled();
Console.WriteLine(path.InterpreterPath);
Guid testId = Guid.NewGuid();
var testDir = TestData.GetTempPath(testId.ToString());
// run the scraper
using (var proc = ProcessOutput.RunHiddenAndCapture(
path.InterpreterPath,
TestData.GetPath("PythonScraper.py"),
testDir,
TestData.GetPath("CompletionDB")
)) {
Console.WriteLine("Command: " + proc.Arguments);
proc.Wait();
// it should succeed
Console.WriteLine("**Stdout**");
foreach (var line in proc.StandardOutputLines) {
Console.WriteLine(line);
}
Console.WriteLine("");
Console.WriteLine("**Stdout**");
foreach (var line in proc.StandardErrorLines) {
Console.WriteLine(line);
}
Assert.AreEqual(0, proc.ExitCode, "Bad exit code: " + proc.ExitCode);
}
// perform some basic validation
dynamic builtinDb = Unpickle.Load(new FileStream(Path.Combine(testDir, path.Version.Is3x() ? "builtins.idb" : "__builtin__.idb"), FileMode.Open, FileAccess.Read));
if (path.Version.Is2x()) { // no open in 3.x
foreach (var overload in builtinDb["members"]["open"]["value"]["overloads"]) {
Assert.AreEqual("__builtin__", overload["ret_type"][0][0]);
Assert.AreEqual("file", overload["ret_type"][0][1]);
}
if (!path.InterpreterPath.Contains("Iron")) {
// http://pytools.codeplex.com/workitem/799
var arr = (IList<object>)builtinDb["members"]["list"]["value"]["members"]["__init__"]["value"]["overloads"];
Assert.AreEqual(
"args",
((dynamic)(arr[0]))["args"][1]["name"]
);
}
}
if (!path.InterpreterPath.Contains("Iron")) {
dynamic itertoolsDb = Unpickle.Load(new FileStream(Path.Combine(testDir, "itertools.idb"), FileMode.Open, FileAccess.Read));
var tee = itertoolsDb["members"]["tee"]["value"];
var overloads = tee["overloads"];
var nArg = overloads[0]["args"][1];
Assert.AreEqual("n", nArg["name"]);
Assert.AreEqual("2", nArg["default_value"]);
dynamic sreDb = Unpickle.Load(new FileStream(Path.Combine(testDir, "_sre.idb"), FileMode.Open, FileAccess.Read));
var members = sreDb["members"];
Assert.IsTrue(members.ContainsKey("SRE_Pattern"));
Assert.IsTrue(members.ContainsKey("SRE_Match"));
}
Console.WriteLine("Passed: {0}", path.InterpreterPath);
}
private static void FullStdLibTest(PythonVersion v) {
v.AssertInstalled();
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
var paths = modules.Select(m => m.LibraryPath).Distinct().ToArray();
bool anySuccess = false;
using (var analyzer = new PythonAnalysis(v.Version)) {
analyzer.SetSearchPaths(paths);
foreach (var modName in modules) {
if (modName.IsCompiled || modName.IsNativeExtension) {
continue;
}
var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName);
if (mod == null) {
Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile));
} else {
anySuccess = true;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
}
