internal ClusterSelector(ClusterEnvironment environment) { InitializeComponent(); _environment = environment; _headNodeCombo.Items.Add("localhost"); if (environment.HeadNode != "localhost") { _headNodeCombo.Items.Add(environment.HeadNode); _headNodeCombo.SelectedIndex = 1; } else { _headNodeCombo.SelectedIndex = 0; _nodesView.Enabled = _manuallySelect.Enabled = _pickNodesCombo.Enabled = _scheduleOneCombo.Enabled = false; } _lastNumProcsValue = environment.NumberOfProcesses.ToString(); _numOfProcsCombo.TextChanged -= NumOfProcsComboChanged; _numOfProcsCombo.Text = _lastNumProcsValue; _numOfProcsCombo.TextChanged += NumOfProcsComboChanged; _scheduleOneCombo.SelectedIndexChanged -= ScheduleOneComboSelectedIndexChanged; _scheduleOneCombo.SelectedIndex = (int)environment.ScheduleProcessPer; _scheduleOneCombo.SelectedIndexChanged += ScheduleOneComboSelectedIndexChanged; if (environment.SelectedNodes == null) { this._nodesView.CheckBoxes = false; } else { this._manuallySelect.Checked = true; } }
private string GetDebugOptions(ClusterEnvironment clusterEnv) { string options = ""; if (_pyService.DebuggerOptions.TeeStandardOutput) { options = RedirectOutputSetting + "=True"; } if (_pyService.DebuggerOptions.BreakOnSystemExitZero) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += BreakSystemExitZero + "=True"; } if (_pyService.DebuggerOptions.DebugStdLib) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += DebugStdLib + "=True"; } if (clusterEnv.HeadNode == "localhost") // don't wait on the cluster, there's no one to press enter. { if (_pyService.DebuggerOptions.WaitOnAbnormalExit) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += WaitOnAbnormalExitSetting + "=True"; } if (_pyService.DebuggerOptions.WaitOnNormalExit) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += WaitOnNormalExitSetting + "=True"; } } var interpArgs = _project.GetProperty(PythonConstants.InterpreterArgumentsSetting); if (!String.IsNullOrWhiteSpace(interpArgs)) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += InterpreterOptions + "=" + interpArgs.Replace(";", ";;"); } return(options); }
private ISchedulerJob CreateSchedulerJob(string commandLine, ClusterEnvironment clusterEnv, Scheduler scheduler, bool debug) { var job = scheduler.CreateJob(); job.IsExclusive = true; job.Name = debug ? "Python MPI Debugging Session" : "Python MPI Session"; string mpiExecCommand, deploymentDir; if (!TryGetMpiExecCommand(clusterEnv.HeadNode, out mpiExecCommand) || !TryGetDeploymentDir(out deploymentDir)) { // unreachable, we've already built the command line (which gets the MPI exec command as well) and checked that we could get the working throw new InvalidOperationException(); } string workingDir = GetWorkingDir(clusterEnv); ISchedulerTask[] tasks; var runTask = job.CreateTask(); runTask.Name = debug ? "Debug MPI Task" : "Run MPI Task"; if (!IsPathUnc(workingDir)) { // we need to copy the files locally onto the cluster var copyTask = job.CreateTask(); copyTask.Name = "Copy files"; copyTask.Type = TaskType.NodePrep; var cleanupTask = job.CreateTask(); cleanupTask.Name = "Cleanup files"; tasks = new[] { copyTask, runTask, cleanupTask }; copyTask.CommandLine = String.Format("\"%WINDIR%\\System32\\cmd.exe\" /c mkdir \"{1}\" & robocopy /s \"{0}\" \"{1}\" & if not errorlevel 2 exit 0", deploymentDir, workingDir); cleanupTask.CommandLine = String.Format("\"%WINDIR%\\System32\\cmd.exe\" /c rmdir /s /q \"{0}\"", workingDir); cleanupTask.Type = TaskType.NodeRelease; } else { tasks = new[] { runTask }; } runTask.CommandLine = commandLine; runTask.WorkDirectory = workingDir; job.NodeGroups.Add(clusterEnv.PickNodesFrom); SetJobParameters(clusterEnv, job, tasks); job.AddTasks(tasks); job.OnJobState += JobStateChanged; AddJobStateStatusUpdate(job, tasks); return(job); }
public override object EditValue(System.ComponentModel.ITypeDescriptorContext context, IServiceProvider provider, object value) { var editSvc = (IWindowsFormsEditorService)provider.GetService(typeof(IWindowsFormsEditorService)); ClusterEnvironment env = (ClusterEnvironment)value; ClusterSelector selector = new ClusterSelector(env); if (editSvc.ShowDialog(selector) == DialogResult.OK) { return(new ClusterEnvironment(selector.Description)); } return(value); }
/// <summary> /// Builds the command line that each task should execute. /// </summary> private bool TryBuildCommandLine(bool debug, ClusterEnvironment environment, string startupFile, string workingDir, out string exe, out string arguments) { exe = arguments = null; string schedulerNode = environment.HeadNode; var appCommand = _project.GetProperty(ClusterOptions.AppCommandSetting); if (String.IsNullOrWhiteSpace(appCommand)) { MessageBox.Show("The path to the interpreter on the cluster is not configured. Please update project properties->Debug->Python Interpreter to point at the correct location of the interpreter", "Python Tools for Visual Studio"); return(false); } string mpiExecCommand; if (!TryGetMpiExecCommand(schedulerNode, out mpiExecCommand)) { return(false); } var appArgs = _project.GetProperty(ClusterOptions.AppArgumentsSetting); if (!String.IsNullOrWhiteSpace(workingDir)) { startupFile = Path.Combine(workingDir, startupFile); } exe = "\"" + mpiExecCommand + "\""; arguments = ""; if (schedulerNode == "localhost") { arguments = "-n " + environment.NumberOfProcesses + " "; } string tmpArgs = "\"" + appCommand + "\" \"" + startupFile + "\" \"" + appArgs + "\""; if (debug) { arguments += "\"" + Path.Combine(workingDir, MpiShimExe) + "\" " + ((IPEndPoint)_listenerSocket.LocalEndPoint).Port + " " + _authGuid.ToString() + " " + GetLocalIPv4Address(schedulerNode) + " " + "\"" + FixDir(workingDir) + "\" " + "\"" + FixDir(_project.ProjectDirectory) + "\" " + "\"" + GetDebugOptions(environment) + "\" " + tmpArgs; } else { arguments += tmpArgs; } return(true); }
/// <summary> /// Publishes the project if the user has configured the publish on run. /// </summary> private bool TryPublishProject(ClusterEnvironment environment, string publishOverrideUrl) { if (_project.PublishBeforeRun() || environment.HeadNode == "localhost") { string msg = null; try { var vsInstallDir = (string)HpcSupportPackage.Instance.ApplicationRegistryRoot.GetValue("InstallDir"); List <IPublishFile> allFiles = new List <IPublishFile>(); // add python debugger files string pyInstallDir = GetPythonToolsInstallPath(); foreach (var file in _pyDebuggerFiles) { allFiles.Add(new CopyFile(Path.Combine(pyInstallDir, file), file)); } string pyHpcInstallDir = GetPythonHpcToolsInstallPath(); foreach (var file in _hpcDebuggerFiles) { allFiles.Add(new CopyFile(Path.Combine(pyHpcInstallDir, file), file)); } // add VS components that we need to run foreach (var file in _vsAssemblies) { allFiles.Add(new CopyFile(file, Path.GetFileName(file))); } // Add vs remote debugger components. string basePath = Path.Combine(Path.Combine(vsInstallDir, "Remote Debugger"), _project.TargetPlatform().ToString()) + "\\"; foreach (var file in Directory.GetFiles(basePath)) { allFiles.Add(new CopyFile(file, file.Substring(basePath.Length))); } if (!_project.Publish(new PublishProjectOptions(allFiles.ToArray(), publishOverrideUrl))) { msg = "Publishing not configured or unknown publishing schema"; } } catch (PublishFailedException e) { msg = e.InnerException.Message; } if (msg != null && CancelLaunch(msg)) { return(false); } } return(true); }
private string GetWorkingDir(ClusterEnvironment clusterEnv) { string workingDir = _project.GetProperty(ClusterOptions.WorkingDirSetting); if (String.IsNullOrWhiteSpace(workingDir)) { if (clusterEnv.HeadNode == "localhost") { return(_project.GetWorkingDirectory()); } return("%TEMP%\\%USERNAME%\\" + _project.ProjectName); } return(workingDir); }
private static void SelectClusterNodes(ClusterEnvironment clusterEnv, ISchedulerJob job, ISchedulerTask[] tasks) { foreach (var node in clusterEnv.SelectedNodes) { foreach (var task in tasks) { if (ShouldSetTaskParameters(task)) { task.RequiredNodes.Add(node); } } } switch (clusterEnv.ScheduleProcessPer) { case ScheduleProcessPer.Socket: job.MinimumNumberOfSockets = job.MaximumNumberOfSockets = clusterEnv.NumberOfProcesses; foreach (var task in tasks) { if (ShouldSetTaskParameters(task)) { task.MinimumNumberOfSockets = task.MaximumNumberOfSockets = clusterEnv.NumberOfProcesses; } } break; case ScheduleProcessPer.Node: job.MinimumNumberOfNodes = job.MaximumNumberOfNodes = clusterEnv.NumberOfProcesses; foreach (var task in tasks) { if (ShouldSetTaskParameters(task)) { task.MinimumNumberOfNodes = task.MaximumNumberOfNodes = clusterEnv.NumberOfProcesses; } } break; case ScheduleProcessPer.Core: job.MinimumNumberOfCores = job.MaximumNumberOfCores = clusterEnv.NumberOfProcesses; foreach (var task in tasks) { if (ShouldSetTaskParameters(task)) { task.MinimumNumberOfCores = task.MaximumNumberOfCores = clusterEnv.NumberOfProcesses; } } break; } }
private static void ScheduleJob(ClusterEnvironment clusterEnv, ISchedulerJob job) { var jobPath = Path.Combine(Environment.GetEnvironmentVariable("CCP_HOME"), "Bin\\job.exe"); var info = new ProcessStartInfo(jobPath, "submit /id:" + job.Id + " /scheduler:" + clusterEnv.HeadNode); var outWin = (IVsOutputWindow)CommonPackage.GetGlobalService(typeof(IVsOutputWindow)); IVsOutputWindowPane pane; if (ErrorHandler.Succeeded(outWin.GetPane(VSConstants.GUID_OutWindowGeneralPane, out pane))) { pane.Activate(); pane.OutputString("Submitting job using command " + info.FileName + " " + info.Arguments + Environment.NewLine); } LaunchRedirectedToVsOutputWindow(info); }
private void UpdateEnvironment() { string[] checkedItems = null; if (_manuallySelect.Checked) { List <string> checks = new List <string>(); foreach (ListViewItem item in _nodesView.Items) { if (item.Checked) { checks.Add(item.Text); } } checkedItems = checks.ToArray(); } _environment = new ClusterEnvironment(_headNodeCombo.Text, (int)UInt32.Parse(_numOfProcsCombo.Text), (ScheduleProcessPer)_scheduleOneCombo.SelectedIndex, _pickNodesCombo.Text, checkedItems ); }
private void UpdateEnvironment() { string[] checkedItems = null; if (_manuallySelect.Checked) { List<string> checks = new List<string>(); foreach (ListViewItem item in _nodesView.Items) { if (item.Checked) { checks.Add(item.Text); } } checkedItems = checks.ToArray(); } _environment = new ClusterEnvironment(_headNodeCombo.Text, (int)UInt32.Parse(_numOfProcsCombo.Text), (ScheduleProcessPer)_scheduleOneCombo.SelectedIndex, _pickNodesCombo.Text, checkedItems ); }
private string GetWorkingDir(ClusterEnvironment clusterEnv) { string workingDir = _project.GetProperty(ClusterOptions.WorkingDirSetting); if (String.IsNullOrWhiteSpace(workingDir)) { if (clusterEnv.HeadNode == "localhost") { return _project.GetWorkingDirectory(); } return "%TEMP%\\%USERNAME%\\" + _project.ProjectName; } return workingDir; }
/// <summary> /// Publishes the project if the user has configured the publish on run. /// </summary> private bool TryPublishProject(ClusterEnvironment environment, string publishOverrideUrl) { if (_project.PublishBeforeRun() || environment.HeadNode == "localhost") { string msg = null; try { var vsInstallDir = (string)HpcSupportPackage.Instance.ApplicationRegistryRoot.GetValue("InstallDir"); List<IPublishFile> allFiles = new List<IPublishFile>(); // add python debugger files string pyInstallDir = Path.GetDirectoryName(PythonToolsInstallPath.GetFile(_pyDebuggerFiles[0])); foreach (var file in _pyDebuggerFiles) { allFiles.Add(new CopyFile(Path.Combine(pyInstallDir, file), file)); } string pyHpcInstallDir = Path.GetDirectoryName(PythonToolsInstallPath.GetFile(MpiShimExe)); foreach (var file in _hpcDebuggerFiles) { allFiles.Add(new CopyFile(Path.Combine(pyHpcInstallDir, file), file)); } // add VS components that we need to run foreach (var file in _vsAssemblies) { allFiles.Add(new CopyFile(file, Path.GetFileName(file))); } // Add vs remote debugger components. string basePath = Path.Combine(Path.Combine(vsInstallDir, "Remote Debugger"), _project.TargetPlatform().ToString()); if (!Directory.Exists(basePath)) { basePath = Path.Combine( (string)HpcSupportPackage.Instance.ApplicationRegistryRoot.GetValue("ShellFolder", ""), "Common7", "Packages", "Debugger", _project.TargetPlatform().ToString() ); } basePath += "\\"; foreach (var file in Directory.GetFiles(basePath, "*", SearchOption.AllDirectories)) { allFiles.Add(new CopyFile(file, file.Substring(basePath.Length))); } if (!_project.Publish(new PublishProjectOptions(allFiles.ToArray(), publishOverrideUrl))) { msg = "Publishing not configured or unknown publishing schema"; } } catch (PublishFailedException e) { msg = e.InnerException.Message; } if (msg != null && CancelLaunch(msg)) { return false; } } return true; }
/// <summary> /// Builds the command line that each task should execute. /// </summary> private bool TryBuildCommandLine(bool debug, ClusterEnvironment environment, string startupFile, string workingDir, out string exe, out string arguments) { exe = arguments = null; string schedulerNode = environment.HeadNode; var appCommand = _project.GetProperty(ClusterOptions.AppCommandSetting); if (String.IsNullOrWhiteSpace(appCommand)) { MessageBox.Show("The path to the interpreter on the cluster is not configured. Please update project properties->Debug->Python Interpreter to point at the correct location of the interpreter", "Python Tools for Visual Studio"); return false; } string mpiExecCommand; if (!TryGetMpiExecCommand(schedulerNode, out mpiExecCommand)) { return false; } var appArgs = _project.GetProperty(ClusterOptions.AppArgumentsSetting); if (!String.IsNullOrWhiteSpace(workingDir)) { startupFile = Path.Combine(workingDir, startupFile); } exe = "\"" + mpiExecCommand + "\""; arguments = ""; if (schedulerNode == "localhost") { arguments = "-n " + environment.NumberOfProcesses + " "; } string tmpArgs = "\"" + appCommand + "\" \"" + startupFile + "\" \"" + appArgs + "\""; if (debug) { arguments += "\"" + Path.Combine(workingDir, MpiShimExe) + "\" " + ((IPEndPoint)_listenerSocket.LocalEndPoint).Port + " " + _authGuid.ToString() + " " + GetLocalIPv4Address(schedulerNode) + " " + "\"" + FixDir(workingDir) + "\" " + "\"" + FixDir(_project.ProjectDirectory) + "\" " + "\"" + GetDebugOptions(environment) + "\" " + tmpArgs; } else { arguments += tmpArgs; } return true; }
private string GetDebugOptions(ClusterEnvironment clusterEnv) { string options = ""; if (_pyService.DebuggerOptions.TeeStandardOutput) { options = RedirectOutputSetting + "=True"; } if (_pyService.DebuggerOptions.BreakOnSystemExitZero) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += BreakSystemExitZero + "=True"; } if (_pyService.DebuggerOptions.DebugStdLib) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += DebugStdLib + "=True"; } if (clusterEnv.HeadNode == "localhost") { // don't wait on the cluster, there's no one to press enter. if (_pyService.DebuggerOptions.WaitOnAbnormalExit) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += WaitOnAbnormalExitSetting + "=True"; } if (_pyService.DebuggerOptions.WaitOnNormalExit) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += WaitOnNormalExitSetting + "=True"; } } var interpArgs = _project.GetProperty(PythonConstants.InterpreterArgumentsSetting); if (!String.IsNullOrWhiteSpace(interpArgs)) { if (!String.IsNullOrEmpty(options)) { options += ";"; } options += InterpreterOptions + "=" + interpArgs.Replace(";", ";;"); } return options; }
public int LaunchFile(string filename, bool debug) { var clusterEnv = new ClusterEnvironment(_project.GetProperty(ClusterOptions.RunEnvironmentSetting)); if (debug) { EnsureHiddenForm(); EnsureListenerThread(); } string workingDir, publishUrl; if (clusterEnv.HeadNode == "localhost") { workingDir = _project.GetProperty(ClusterOptions.WorkingDirSetting); if (String.IsNullOrWhiteSpace(workingDir)) { workingDir = Path.Combine(Path.GetTempPath(), "HpcPyDebug" + Guid.NewGuid().ToString()); } if (!Directory.Exists(workingDir)) { Directory.CreateDirectory(workingDir); } publishUrl = "file://" + workingDir; } else { publishUrl = null; workingDir = GetWorkingDir(clusterEnv); // make sure we have a valid deployement dir as well string deploymentDir; if (!TryGetDeploymentDir(out deploymentDir)) { return(VSConstants.S_OK); } } string exe, arguments; if (!TryBuildCommandLine(debug, clusterEnv, filename, workingDir, out exe, out arguments) || !TryPublishProject(clusterEnv, publishUrl)) { return(VSConstants.S_OK); } if (clusterEnv.HeadNode == "localhost") { // run locally var startInfo = new ProcessStartInfo(exe, arguments); var proc = LaunchRedirectedToVsOutputWindow(startInfo, false); } else { EnsureGeneralPane(); var commandLine = exe + " " + arguments; var scheduler = new Scheduler(); scheduler.Connect(clusterEnv.HeadNode); var job = CreateSchedulerJob(commandLine, clusterEnv, scheduler, debug); scheduler.AddJob(job); SetStatus("Scheduling job on server..."); ScheduleJob(clusterEnv, job); } return(VSConstants.S_OK); }
/// <summary> /// Called to initially set the environment /// </summary> /// <param name="env"></param> internal void LoadRunEnvironment(ClusterEnvironment env) { _env = env; }
private ISchedulerJob CreateSchedulerJob(string commandLine, ClusterEnvironment clusterEnv, Scheduler scheduler, bool debug) { var job = scheduler.CreateJob(); job.IsExclusive = true; job.Name = debug ? "Python MPI Debugging Session" : "Python MPI Session"; string mpiExecCommand, deploymentDir; if (!TryGetMpiExecCommand(clusterEnv.HeadNode, out mpiExecCommand) || !TryGetDeploymentDir(out deploymentDir)) { // unreachable, we've already built the command line (which gets the MPI exec command as well) and checked that we could get the working throw new InvalidOperationException(); } string workingDir = GetWorkingDir(clusterEnv); ISchedulerTask[] tasks; var runTask = job.CreateTask(); runTask.Name = debug ? "Debug MPI Task" : "Run MPI Task"; if (!IsPathUnc(workingDir)) { // we need to copy the files locally onto the cluster var copyTask = job.CreateTask(); copyTask.Name = "Copy files"; copyTask.Type = TaskType.NodePrep; var cleanupTask = job.CreateTask(); cleanupTask.Name = "Cleanup files"; tasks = new[] { copyTask, runTask, cleanupTask }; copyTask.CommandLine = String.Format("\"%WINDIR%\\System32\\cmd.exe\" /c mkdir \"{1}\" & robocopy /s \"{0}\" \"{1}\" & if not errorlevel 2 exit 0", deploymentDir, workingDir); cleanupTask.CommandLine = String.Format("\"%WINDIR%\\System32\\cmd.exe\" /c rmdir /s /q \"{0}\"", workingDir); cleanupTask.Type = TaskType.NodeRelease; } else { tasks = new[] { runTask }; } runTask.CommandLine = commandLine; runTask.WorkDirectory = workingDir; job.NodeGroups.Add(clusterEnv.PickNodesFrom); SetJobParameters(clusterEnv, job, tasks); job.AddTasks(tasks); job.OnJobState += JobStateChanged; job.OnTaskState += OnTaskStateChange; return job; }
public int LaunchFile(string filename, bool debug) { var clusterEnv = new ClusterEnvironment(_project.GetProperty(ClusterOptions.RunEnvironmentSetting)); if (debug) { EnsureHiddenForm(); EnsureListenerThread(); } string workingDir, publishUrl; if (clusterEnv.HeadNode == "localhost") { workingDir = _project.GetProperty(ClusterOptions.WorkingDirSetting); if (String.IsNullOrWhiteSpace(workingDir)) { workingDir = Path.Combine(Path.GetTempPath(), "HpcPyDebug" + Guid.NewGuid().ToString()); } if (!Directory.Exists(workingDir)) { Directory.CreateDirectory(workingDir); } publishUrl = "file://" + workingDir; } else { workingDir = GetWorkingDir(clusterEnv); // make sure we have a valid deployement dir as well string deploymentDir; if (!TryGetDeploymentDir(out deploymentDir)) { return VSConstants.S_OK; } publishUrl = deploymentDir; } string exe, arguments; if (!TryBuildCommandLine(debug, clusterEnv, filename, workingDir, out exe, out arguments) || !TryPublishProject(clusterEnv, publishUrl)) { return VSConstants.S_OK; } if (clusterEnv.HeadNode == "localhost") { // run locally if (debug) { var startInfo = new ProcessStartInfo(exe, arguments); LaunchRedirectedToVsOutputWindow(startInfo, false); } else { Process.Start(exe, arguments); } } else { EnsureGeneralPane(); var commandLine = exe + " " + arguments; var scheduler = new Scheduler(); scheduler.Connect(clusterEnv.HeadNode); var job = CreateSchedulerJob(commandLine, clusterEnv, scheduler, debug); scheduler.AddJob(job); SetStatus("Scheduling job on server..."); ScheduleJob(scheduler, job); } return VSConstants.S_OK; }