/// <summary> /// Submit job to scheduler /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobSpecification">Job specification</param> /// <param name="credentials">Credentials</param> /// <returns></returns> public virtual IEnumerable <SubmittedTaskInfo> SubmitJob(object connectorClient, JobSpecification jobSpecification, ClusterAuthenticationCredentials credentials) { var shellCommandSb = new StringBuilder(); SshCommandWrapper command = null; string shellCommand = (string)_convertor.ConvertJobSpecificationToJob(jobSpecification, null); _log.Info($"Submitting job \"{jobSpecification.Id}\", command \"{shellCommand}\""); string sshCommandBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(shellCommand)); command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.ExecutieCmdPath} {sshCommandBase64}"); shellCommandSb.Clear(); //compose command with parameters of job and task IDs shellCommandSb.Append($"{_linuxLocalCommandScripts.RunLocalCmdPath} {jobSpecification.FileTransferMethod.Cluster.LocalBasepath}/{jobSpecification.Id}/"); jobSpecification.Tasks.ForEach(task => shellCommandSb.Append($" {task.Id}")); //log local HPC Run script to log file shellCommandSb.Append($" >> {jobSpecification.FileTransferMethod.Cluster.LocalBasepath}/{jobSpecification.Id}/job_log.txt &"); shellCommand = shellCommandSb.ToString(); sshCommandBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(shellCommand)); command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.ExecutieCmdPath} {sshCommandBase64}"); return(GetActualTasksInfo(connectorClient, jobSpecification.Cluster, new string[] { $"{jobSpecification.Id}" })); }
/// <summary> /// Delete job directory /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> public void DeleteJobDirectory(object connectorClient, SubmittedJobInfo jobInfo) { string shellCommand = $"rm -Rf {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}"; var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), shellCommand); _log.Info($"Job directory \"{jobInfo.Specification.Id}\" was deleted. Result: \"{sshCommand.Result}\""); }
/// <summary> /// Allow direct file transfer acces for user /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="publicKey">Public key</param> /// <param name="jobInfo">Job information</param> public void AllowDirectFileTransferAccessForUserToJob(object connectorClient, string publicKey, SubmittedJobInfo jobInfo) { publicKey = StringUtils.RemoveWhitespace(publicKey); var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.AddFiletransferKeyCmdPath} {publicKey} {jobInfo.Specification.Id}"); _log.InfoFormat($"Allow file transfer result: \"{sshCommand.Result}\""); }
/// <summary> /// Remove direct file transfer acces for user /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="publicKey">Public key</param> public void RemoveDirectFileTransferAccessForUserToJob(object connectorClient, string publicKey) { publicKey = StringUtils.RemoveWhitespace(publicKey); var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.RemoveFiletransferKeyCmdPath} {publicKey}"); _log.Info($"Remove permission for direct file transfer result: \"{sshCommand.Result}\""); }
/// <summary> /// Get allocated nodes per task /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="taskInfo">Task information</param> public virtual IEnumerable <string> GetAllocatedNodes(object connectorClient, SubmittedTaskInfo taskInfo) { SshCommandWrapper command = null; StringBuilder cmdBuilder = new(); var cluster = taskInfo.Specification.JobSpecification.Cluster; var nodeNames = taskInfo.TaskAllocationNodes.Select(s => $"{s.AllocationNodeId}.{cluster.DomainName ?? cluster.MasterNodeName}") .ToList(); nodeNames.ForEach(f => cmdBuilder.Append($"dig +short {f};")); string sshCommand = cmdBuilder.ToString(); _log.Info($"Get allocation nodes of task \"{taskInfo.Id}\", command \"{sshCommand}\""); try { command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand); return(command.Result.Split('\n').Where(w => !string.IsNullOrEmpty(w)) .ToList()); } catch (FormatException e) { throw new Exception($@"Exception thrown when retrieving allocation nodes used by running task (HPC job): ""{taskInfo.ScheduledJobId}"". Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n Command line for job submission: ""{sshCommand}""\n", e); } }
/// <summary> /// Submit job to scheduler /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobSpecification">Job specification</param> /// <param name="credentials">Credentials</param> /// <returns></returns> /// <exception cref="Exception"></exception> public virtual IEnumerable <SubmittedTaskInfo> SubmitJob(object connectorClient, JobSpecification jobSpecification, ClusterAuthenticationCredentials credentials) { var jobIdsWithJobArrayIndexes = new List <string>(); SshCommandWrapper command = null; string sshCommand = (string)_convertor.ConvertJobSpecificationToJob(jobSpecification, "qsub -koed"); _log.Info($"Submitting job \"{jobSpecification.Id}\", command \"{sshCommand}\""); string sshCommandBase64 = $"{_commands.InterpreterCommand} '{_commands.ExecutieCmdScriptPath} {Convert.ToBase64String(Encoding.UTF8.GetBytes(sshCommand))}'"; try { command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommandBase64); var jobIds = _convertor.GetJobIds(command.Result).ToList(); for (int i = 0; i < jobSpecification.Tasks.Count; i++) { jobIdsWithJobArrayIndexes.AddRange(string.IsNullOrEmpty(jobSpecification.Tasks[i].JobArrays) ? new List <string> { jobIds[i] } : CombineScheduledJobIdWithJobArrayIndexes(jobIds[i], jobSpecification.Tasks[i].JobArrays)); } return(GetActualTasksInfo(connectorClient, jobSpecification.Cluster, jobIdsWithJobArrayIndexes)); } catch (FormatException e) { throw new Exception(@$ "Exception thrown when submitting a job: " "{jobSpecification.Name}" " to the cluster: " "{jobSpecification.Cluster.Name}" ". Submission script result: " "{command.Result}" ".\nSubmission script error message: " "{command.Error}" ".\n Command line for job submission: " "{sshCommandBase64}" ".\n", e); } }
/// <summary> /// Get generic command templates parameters from script /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="userScriptPath">Generic script path</param> /// <returns></returns> public virtual IEnumerable <string> GetParametersFromGenericUserScript(object connectorClient, string userScriptPath) { var genericCommandParameters = new List <string>(); string shellCommand = $"cat {userScriptPath}"; var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), shellCommand); foreach (Match match in Regex.Matches(sshCommand.Result, @$ "{_genericCommandKeyParameter}([\s\t]+[A-z_\-]+)\n", RegexOptions.IgnoreCase | RegexOptions.Compiled))
/// <summary> /// Get actual scheduler queue status /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="nodeType">Cluster node type</param> public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType) { SshCommandWrapper command = null; var allocationCluster = string.Empty; if (!string.IsNullOrEmpty(nodeType.ClusterAllocationName)) { allocationCluster = $"--clusters={nodeType.ClusterAllocationName} "; } var sshCommand = $"{_commands.InterpreterCommand} 'sinfo -t alloc {allocationCluster}--partition={nodeType.Queue} -h -o \"%.6D\"'"; _log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{sshCommand}\""); try { command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand); return(_convertor.ReadQueueActualInformation(nodeType, command.Result)); } catch (FormatException e) { throw new Exception($@"Exception thrown when retrieving usage of Cluster node: ""{nodeType.Name}"". Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n Command line for queue usage: ""{sshCommand}""\n", e); } }
/// <summary> /// Copy job data to temp folder /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> /// <param name="hash">Hash</param> public void CopyJobDataFromTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash) { string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}/."; string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}"; var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataFromTempCmdPath} {inputDirectory} {outputDirectory}"); _log.Info($"Temp data \"{hash}\" were copied to job directory \"{jobInfo.Specification.Id}\", result: \"{sshCommand.Result}\""); }
/// <summary> /// Cancel job /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="submitedTasksInfo">Submitted tasks id´s</param> /// <param name="message">Message</param> public virtual void CancelJob(object connectorClient, IEnumerable <SubmittedTaskInfo> submitedTasksInfo, string message) { StringBuilder commandSb = new(); var localClusterJobIds = submitedTasksInfo.Select(s => s.Specification.JobSpecification.Id.ToString()) .Distinct(); localClusterJobIds.ToList().ForEach(id => commandSb.Append($"{_linuxLocalCommandScripts.CancelJobCmdPath} {id};")); string command = commandSb.ToString(); _log.Info($"Cancel jobs \"{string.Join(",", submitedTasksInfo.Select(s => s.ScheduledJobId))}\", command \"{command}\", message \"{message}\""); SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), command); }
/// <summary> /// Copy job data from temp folder /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> /// <param name="hash">Hash</param> public void CopyJobDataToTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash, string path) { //if path is null or empty then all files and directories from ClusterLocalBasepath will be copied to hash directory string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{path}"; inputDirectory += string.IsNullOrEmpty(path) ? "." : string.Empty; string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}"; var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataToTempCmdPath} {inputDirectory} {outputDirectory}"); _log.Info($"Job data \"{jobInfo.Specification.Id}/{path}\" were copied to temp directory \"{hash}\", result: \"{sshCommand.Result}\""); }
/// <summary> /// Create job directory /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> public void CreateJobDirectory(object connectorClient, SubmittedJobInfo jobInfo) { var cmdBuilder = new StringBuilder($"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id};"); foreach (var task in jobInfo.Tasks) { var path = !string.IsNullOrEmpty(task.Specification.ClusterTaskSubdirectory) ? $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id}/{task.Specification.ClusterTaskSubdirectory};" : $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id};"; cmdBuilder.Append(path); } var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), cmdBuilder.ToString()); _log.Info($"Create job directory result: \"{sshCommand.Result.Replace("\n", string.Empty)}\""); }
/// <summary> /// Get cluster node usage /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="nodeType">ClusterNode type</param> /// <returns></returns> public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType) { var usage = new ClusterNodeUsage { NodeType = nodeType }; var command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), _linuxLocalCommandScripts.CountJobsCmdPath); _log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{command}\""); if (int.TryParse(command.Result, out int totalJobs)) { usage.TotalJobs = totalJobs; } return(usage); }
/// <summary> /// Get actual scheduler queue status /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="nodeType">Cluster node type</param> /// <returns></returns> /// <exception cref="Exception"></exception> public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType) { SshCommandWrapper command = null; string sshCommand = $"{_commands.InterpreterCommand} 'qstat -Q -f {nodeType.Queue}'"; _log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{sshCommand}\""); try { command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand); return(_convertor.ReadQueueActualInformation(nodeType, command.Result)); } catch (FormatException e) { throw new Exception($@"Exception thrown when retrieving parameters of queue: ""{nodeType.Name}"". Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n Command line for job submission: ""{sshCommand}""\n", e); } }
/// <summary> /// Cancel job /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="submitedTasksInfo">Submitted tasks id´s</param> /// <param name="message">Message</param> public virtual void CancelJob(object connectorClient, IEnumerable <SubmittedTaskInfo> submitedTasksInfo, string message) { StringBuilder cmdBuilder = new(); try { submitedTasksInfo.ToList().ForEach(f => cmdBuilder.Append($"{_commands.InterpreterCommand} 'qdel {f.ScheduledJobId}';")); string sshCommand = cmdBuilder.ToString(); _log.Info($"Cancel jobs \"{string.Join(",", submitedTasksInfo.Select(s => s.ScheduledJobId))}\", command \"{sshCommand}\", message \"{message}\""); SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand); } catch (SshCommandException ce) { if (!Regex.Match(ce.Message, "qdel: Job has finished", RegexOptions.Compiled).Success) { throw ce; } } }
/// <summary> /// Cancel job /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="submitedTasksInfo">Submitted tasks id´s</param> /// <param name="message">Message</param> public virtual void CancelJob(object connectorClient, IEnumerable <SubmittedTaskInfo> submitedTasksInfo, string message) { StringBuilder cmdBuilder = new(); foreach (var submitedTaskInfo in submitedTasksInfo) { var allocationCluster = string.Empty; if (!string.IsNullOrEmpty(submitedTaskInfo.Specification.ClusterNodeType.ClusterAllocationName)) { allocationCluster = $"-M {submitedTaskInfo.Specification.ClusterNodeType.ClusterAllocationName} "; } cmdBuilder.Append($"{ _commands.InterpreterCommand} 'scancel {allocationCluster}{submitedTaskInfo.ScheduledJobId}';"); } string sshCommand = cmdBuilder.ToString(); _log.Info($"Cancel jobs \"{string.Join(",", submitedTasksInfo.Select(s => s.ScheduledJobId))}\", command \"{sshCommand}\", message \"{message}\""); SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand); }