/// <summary>
/// Submit job to scheduler
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobSpecification">Job specification</param>
/// <param name="credentials">Credentials</param>
/// <returns></returns>
public virtual IEnumerable <SubmittedTaskInfo> SubmitJob(object connectorClient, JobSpecification jobSpecification, ClusterAuthenticationCredentials credentials)
{
var shellCommandSb = new StringBuilder();
SshCommandWrapper command = null;
string shellCommand = (string)_convertor.ConvertJobSpecificationToJob(jobSpecification, null);
_log.Info($"Submitting job \"{jobSpecification.Id}\", command \"{shellCommand}\"");
string sshCommandBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(shellCommand));
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.ExecutieCmdPath} {sshCommandBase64}");
shellCommandSb.Clear();
//compose command with parameters of job and task IDs
shellCommandSb.Append($"{_linuxLocalCommandScripts.RunLocalCmdPath} {jobSpecification.FileTransferMethod.Cluster.LocalBasepath}/{jobSpecification.Id}/");
jobSpecification.Tasks.ForEach(task => shellCommandSb.Append($" {task.Id}"));
//log local HPC Run script to log file
shellCommandSb.Append($" >> {jobSpecification.FileTransferMethod.Cluster.LocalBasepath}/{jobSpecification.Id}/job_log.txt &");
shellCommand = shellCommandSb.ToString();
sshCommandBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(shellCommand));
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.ExecutieCmdPath} {sshCommandBase64}");
return(GetActualTasksInfo(connectorClient, jobSpecification.Cluster, new string[] { $"{jobSpecification.Id}" }));
}
/// <summary>
/// Delete job directory
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
public void DeleteJobDirectory(object connectorClient, SubmittedJobInfo jobInfo)
{
string shellCommand = $"rm -Rf {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}";
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), shellCommand);
_log.Info($"Job directory \"{jobInfo.Specification.Id}\" was deleted. Result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Allow direct file transfer acces for user
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="publicKey">Public key</param>
/// <param name="jobInfo">Job information</param>
public void AllowDirectFileTransferAccessForUserToJob(object connectorClient, string publicKey, SubmittedJobInfo jobInfo)
{
publicKey = StringUtils.RemoveWhitespace(publicKey);
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.AddFiletransferKeyCmdPath} {publicKey} {jobInfo.Specification.Id}");
_log.InfoFormat($"Allow file transfer result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Remove direct file transfer acces for user
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="publicKey">Public key</param>
public void RemoveDirectFileTransferAccessForUserToJob(object connectorClient, string publicKey)
{
publicKey = StringUtils.RemoveWhitespace(publicKey);
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.RemoveFiletransferKeyCmdPath} {publicKey}");
_log.Info($"Remove permission for direct file transfer result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Get allocated nodes per task
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="taskInfo">Task information</param>
public virtual IEnumerable <string> GetAllocatedNodes(object connectorClient, SubmittedTaskInfo taskInfo)
{
SshCommandWrapper command = null;
StringBuilder cmdBuilder = new();
var cluster = taskInfo.Specification.JobSpecification.Cluster;
var nodeNames = taskInfo.TaskAllocationNodes.Select(s => $"{s.AllocationNodeId}.{cluster.DomainName ?? cluster.MasterNodeName}")
.ToList();
nodeNames.ForEach(f => cmdBuilder.Append($"dig +short {f};"));
string sshCommand = cmdBuilder.ToString();
_log.Info($"Get allocation nodes of task \"{taskInfo.Id}\", command \"{sshCommand}\"");
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
return(command.Result.Split('\n').Where(w => !string.IsNullOrEmpty(w))
.ToList());
}
catch (FormatException e)
{
throw new Exception($@"Exception thrown when retrieving allocation nodes used by running task (HPC job): ""{taskInfo.ScheduledJobId}"".
Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n
Command line for job submission: ""{sshCommand}""\n", e);
}
}
/// <summary>
/// Submit job to scheduler
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobSpecification">Job specification</param>
/// <param name="credentials">Credentials</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public virtual IEnumerable <SubmittedTaskInfo> SubmitJob(object connectorClient, JobSpecification jobSpecification, ClusterAuthenticationCredentials credentials)
{
var jobIdsWithJobArrayIndexes = new List <string>();
SshCommandWrapper command = null;
string sshCommand = (string)_convertor.ConvertJobSpecificationToJob(jobSpecification, "qsub -koed");
_log.Info($"Submitting job \"{jobSpecification.Id}\", command \"{sshCommand}\"");
string sshCommandBase64 = $"{_commands.InterpreterCommand} '{_commands.ExecutieCmdScriptPath} {Convert.ToBase64String(Encoding.UTF8.GetBytes(sshCommand))}'";
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommandBase64);
var jobIds = _convertor.GetJobIds(command.Result).ToList();
for (int i = 0; i < jobSpecification.Tasks.Count; i++)
{
jobIdsWithJobArrayIndexes.AddRange(string.IsNullOrEmpty(jobSpecification.Tasks[i].JobArrays)
? new List <string> {
jobIds[i]
}
: CombineScheduledJobIdWithJobArrayIndexes(jobIds[i], jobSpecification.Tasks[i].JobArrays));
}
return(GetActualTasksInfo(connectorClient, jobSpecification.Cluster, jobIdsWithJobArrayIndexes));
}
catch (FormatException e)
{
throw new Exception(@$ "Exception thrown when submitting a job: " "{jobSpecification.Name}" " to the cluster: " "{jobSpecification.Cluster.Name}" ".
Submission script result: " "{command.Result}" ".\nSubmission script error message: " "{command.Error}" ".\n
Command line for job submission: " "{sshCommandBase64}" ".\n", e);
}
}
/// <summary>
/// Get generic command templates parameters from script
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="userScriptPath">Generic script path</param>
/// <returns></returns>
public virtual IEnumerable <string> GetParametersFromGenericUserScript(object connectorClient, string userScriptPath)
{
var genericCommandParameters = new List <string>();
string shellCommand = $"cat {userScriptPath}";
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), shellCommand);
foreach (Match match in Regex.Matches(sshCommand.Result, @$ "{_genericCommandKeyParameter}([\s\t]+[A-z_\-]+)\n", RegexOptions.IgnoreCase | RegexOptions.Compiled))
/// <summary>
/// Get actual scheduler queue status
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="nodeType">Cluster node type</param>
public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType)
{
SshCommandWrapper command = null;
var allocationCluster = string.Empty;
if (!string.IsNullOrEmpty(nodeType.ClusterAllocationName))
{
allocationCluster = $"--clusters={nodeType.ClusterAllocationName} ";
}
var sshCommand = $"{_commands.InterpreterCommand} 'sinfo -t alloc {allocationCluster}--partition={nodeType.Queue} -h -o \"%.6D\"'";
_log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{sshCommand}\"");
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
return(_convertor.ReadQueueActualInformation(nodeType, command.Result));
}
catch (FormatException e)
{
throw new Exception($@"Exception thrown when retrieving usage of Cluster node: ""{nodeType.Name}"".
Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n
Command line for queue usage: ""{sshCommand}""\n", e);
}
}
/// <summary>
/// Copy job data to temp folder
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
/// <param name="hash">Hash</param>
public void CopyJobDataFromTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash)
{
string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}/.";
string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}";
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataFromTempCmdPath} {inputDirectory} {outputDirectory}");
_log.Info($"Temp data \"{hash}\" were copied to job directory \"{jobInfo.Specification.Id}\", result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Cancel job
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="submitedTasksInfo">Submitted tasks id´s</param>
/// <param name="message">Message</param>
public virtual void CancelJob(object connectorClient, IEnumerable <SubmittedTaskInfo> submitedTasksInfo, string message)
{
StringBuilder commandSb = new();
var localClusterJobIds = submitedTasksInfo.Select(s => s.Specification.JobSpecification.Id.ToString())
.Distinct();
localClusterJobIds.ToList().ForEach(id => commandSb.Append($"{_linuxLocalCommandScripts.CancelJobCmdPath} {id};"));
string command = commandSb.ToString();
_log.Info($"Cancel jobs \"{string.Join(",", submitedTasksInfo.Select(s => s.ScheduledJobId))}\", command \"{command}\", message \"{message}\"");
SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), command);
}
/// <summary>
/// Copy job data from temp folder
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
/// <param name="hash">Hash</param>
public void CopyJobDataToTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash, string path)
{
//if path is null or empty then all files and directories from ClusterLocalBasepath will be copied to hash directory
string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{path}";
inputDirectory += string.IsNullOrEmpty(path) ? "." : string.Empty;
string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}";
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataToTempCmdPath} {inputDirectory} {outputDirectory}");
_log.Info($"Job data \"{jobInfo.Specification.Id}/{path}\" were copied to temp directory \"{hash}\", result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Create job directory
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
public void CreateJobDirectory(object connectorClient, SubmittedJobInfo jobInfo)
{
var cmdBuilder = new StringBuilder($"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id};");
foreach (var task in jobInfo.Tasks)
{
var path = !string.IsNullOrEmpty(task.Specification.ClusterTaskSubdirectory)
? $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id}/{task.Specification.ClusterTaskSubdirectory};"
: $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id};";
cmdBuilder.Append(path);
}
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), cmdBuilder.ToString());
_log.Info($"Create job directory result: \"{sshCommand.Result.Replace("\n", string.Empty)}\"");
}
/// <summary>
/// Get cluster node usage
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="nodeType">ClusterNode type</param>
/// <returns></returns>
public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType)
{
var usage = new ClusterNodeUsage
{
NodeType = nodeType
};
var command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), _linuxLocalCommandScripts.CountJobsCmdPath);
_log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{command}\"");
if (int.TryParse(command.Result, out int totalJobs))
{
usage.TotalJobs = totalJobs;
}
return(usage);
}
/// <summary>
/// Get actual scheduler queue status
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="nodeType">Cluster node type</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType)
{
SshCommandWrapper command = null;
string sshCommand = $"{_commands.InterpreterCommand} 'qstat -Q -f {nodeType.Queue}'";
_log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{sshCommand}\"");
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
return(_convertor.ReadQueueActualInformation(nodeType, command.Result));
}
catch (FormatException e)
{
throw new Exception($@"Exception thrown when retrieving parameters of queue: ""{nodeType.Name}"".
Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n
Command line for job submission: ""{sshCommand}""\n", e);
}
}
/// <summary>
/// Cancel job
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="submitedTasksInfo">Submitted tasks id´s</param>
/// <param name="message">Message</param>
public virtual void CancelJob(object connectorClient, IEnumerable <SubmittedTaskInfo> submitedTasksInfo, string message)
{
StringBuilder cmdBuilder = new();
try
{
submitedTasksInfo.ToList().ForEach(f => cmdBuilder.Append($"{_commands.InterpreterCommand} 'qdel {f.ScheduledJobId}';"));
string sshCommand = cmdBuilder.ToString();
_log.Info($"Cancel jobs \"{string.Join(",", submitedTasksInfo.Select(s => s.ScheduledJobId))}\", command \"{sshCommand}\", message \"{message}\"");
SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
}
catch (SshCommandException ce)
{
if (!Regex.Match(ce.Message, "qdel: Job has finished", RegexOptions.Compiled).Success)
{
throw ce;
}
}
}
/// <summary>
/// Cancel job
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="submitedTasksInfo">Submitted tasks id´s</param>
/// <param name="message">Message</param>
public virtual void CancelJob(object connectorClient, IEnumerable <SubmittedTaskInfo> submitedTasksInfo, string message)
{
StringBuilder cmdBuilder = new();
foreach (var submitedTaskInfo in submitedTasksInfo)
{
var allocationCluster = string.Empty;
if (!string.IsNullOrEmpty(submitedTaskInfo.Specification.ClusterNodeType.ClusterAllocationName))
{
allocationCluster = $"-M {submitedTaskInfo.Specification.ClusterNodeType.ClusterAllocationName} ";
}
cmdBuilder.Append($"{ _commands.InterpreterCommand} 'scancel {allocationCluster}{submitedTaskInfo.ScheduledJobId}';");
}
string sshCommand = cmdBuilder.ToString();
_log.Info($"Cancel jobs \"{string.Join(",", submitedTasksInfo.Select(s => s.ScheduledJobId))}\", command \"{sshCommand}\", message \"{message}\"");
SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
}