/// <summary>
/// Create tunnel
/// </summary>
/// <param name="taskInfo">Task info</param>
/// <param name="nodeHost">Cluster node address</param>
/// <param name="nodePort">Cluster node port</param>
public void CreateTunnel(SubmittedTaskInfo taskInfo, string nodeHost, int nodePort)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(taskInfo.Specification.JobSpecification.ClusterUser);
try
{
_adapter.CreateTunnel(schedulerConnection.Connection, taskInfo, nodeHost, nodePort);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
/// <summary>
/// Remove tunnel
/// </summary>
/// <param name="taskInfo">Task info</param>
public void RemoveTunnel(SubmittedTaskInfo taskInfo)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(taskInfo.Specification.JobSpecification.ClusterUser);
try
{
_adapter.RemoveTunnel(schedulerConnection, taskInfo);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
/// <summary>
/// Get allocated nodes address for task
/// </summary>
/// <param name="taskInfo">Task information</param>
public IEnumerable <string> GetAllocatedNodes(SubmittedTaskInfo taskInfo)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(taskInfo.Specification.JobSpecification.ClusterUser);
try
{
return(_adapter.GetAllocatedNodes(schedulerConnection.Connection, taskInfo));
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
/// <summary>
/// Calculate used resources for task
/// </summary>
/// <param name="task">Task</param>
/// <returns></returns>
private static double?CalculateUsedResourcesForTask(SubmittedTaskInfo task)
{
if (task.State >= TaskState.Running)
{
double walltimeInSeconds = task.AllocatedTime ?? 0;
int ncpus = task.AllocatedCores ?? task.Specification.MaxCores ?? 0;
return(Math.Round((walltimeInSeconds * ncpus) / 3600, 3));
}
else
{
return(null);
}
}
/// <summary>
/// Convert Task To TaskInfo
/// </summary>
/// <param name="jobDTO">Scheduler job information</param>
/// <returns></returns>
public override SubmittedTaskInfo ConvertTaskToTaskInfo(ISchedulerJobInfo jobDTO)
{
var taskInfo = new SubmittedTaskInfo();
taskInfo.ScheduledJobId = jobDTO.SchedulerJobId.ToString();
taskInfo.Name = jobDTO.Name;
taskInfo.State = jobDTO.TaskState;
taskInfo.StartTime = jobDTO.StartTime;
taskInfo.EndTime = jobDTO.EndTime;
taskInfo.AllocatedTime = jobDTO.AllocatedTime.TotalSeconds;
taskInfo.TaskAllocationNodes = taskInfo.TaskAllocationNodes.Select(s => new SubmittedTaskAllocationNodeInfo
{
AllocationNodeId = s.ToString(),
SubmittedTaskInfoId = long.Parse(taskInfo.Name)
})
.ToList();
taskInfo.AllParameters = jobDTO.SchedulerResponseParameters;
return(taskInfo);
}
/// <summary>
/// Convert submitted task to object for usage reporting
/// </summary>
/// <param name="task">Task</param>
/// <returns></returns>
internal static SubmittedTaskInfoUsageReport ConvertToUsageReport(this SubmittedTaskInfo task)
{
var taskInfoUsageReport = new SubmittedTaskInfoUsageReport
{
Id = task.Id,
Name = task.Name,
Priority = task.Priority,
State = task.State,
CpuHyperThreading = task.CpuHyperThreading ?? false,
ScheduledJobId = task.ScheduledJobId,
CommandTemplateId = task.Specification.CommandTemplate.Id,
AllocatedTime = task.AllocatedTime,
StartTime = task.StartTime,
Usage = CalculateUsedResourcesForTask(task),
EndTime = task.EndTime
};
return(taskInfoUsageReport);
}
private static SubmittedTaskInfoExt ConvertIntToExt(this SubmittedTaskInfo task)
{
SubmittedTaskInfoExt convert = new()
{
Id = task.Id,
Name = task.Name,
State = task.State.ConvertIntToExt(),
Priority = task.Priority.ConvertIntToExt(),
AllocatedTime = task.AllocatedTime,
AllocatedCoreIds = task.TaskAllocationNodes?.Select(s => s.AllocationNodeId)
.ToArray(),
StartTime = task.StartTime,
EndTime = task.EndTime,
CpuHyperThreading = task.CpuHyperThreading,
ErrorMessage = task.ErrorMessage,
NodeType = task.NodeType?.ConvertIntToExt()
};
return(convert);
}
public virtual ICollection <JobFileContent> DownloadPartOfJobFileFromCluster(SubmittedTaskInfo taskInfo, SynchronizableFiles fileType, long offset)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, taskInfo.Specification.JobSpecification);
string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification);
FullFileSpecification fileInfo = CreateSynchronizableFileInfoForType(taskInfo.Specification, taskClusterDirectoryPath, fileType);
IFileSynchronizer synchronizer = CreateFileSynchronizer(fileInfo, taskInfo.Specification.JobSpecification.ClusterUser);
synchronizer.Offset = offset;
synchronizer.SyncFileInfo.DestinationDirectory = null;
ICollection <JobFileContent> result = synchronizer.SynchronizeFiles();
if (result != null)
{
foreach (JobFileContent content in result)
{
content.FileType = fileType;
content.SubmittedTaskInfoId = taskInfo.Id;
}
}
return(result);
}
/// <summary>
/// Get tunnels information
/// </summary>
/// <param name="taskInfo">Task info</param>
/// <param name="nodeHost">Node host</param>
/// <returns></returns>
public IEnumerable <TunnelInfo> GetTunnelsInfos(SubmittedTaskInfo taskInfo, string nodeHost)
{
return(_adapter.GetTunnelsInfos(taskInfo, nodeHost));
}
/// <summary>
/// Get allocated nodes per task
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="taskInfo">Task information</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public virtual IEnumerable <string> GetAllocatedNodes(object connectorClient, SubmittedTaskInfo taskInfo)
{
List <string> allocatedNodes = new();
StringBuilder allocationNodeSb = new();
allocationNodeSb.Clear();
allocationNodeSb.Append(taskInfo.Specification.ClusterNodeType.Cluster.DomainName ?? LocalDomainName);
if (taskInfo.NodeType.Cluster.Port.HasValue)
{
allocationNodeSb.Append($":{taskInfo.NodeType.Cluster.Port.Value}");
}
allocatedNodes.Add(allocationNodeSb.ToString());
_log.Info($"Get allocation nodes of task \"{taskInfo.Id}\"");
return(allocatedNodes.Distinct());
}
/// <summary>
/// Get allocated nodes per task
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="taskInfo">Task information</param>
public virtual IEnumerable <string> GetAllocatedNodes(object connectorClient, SubmittedTaskInfo taskInfo)
{
SshCommandWrapper command = null;
StringBuilder cmdBuilder = new();
var cluster = taskInfo.Specification.JobSpecification.Cluster;
var nodeNames = taskInfo.TaskAllocationNodes.Select(s => $"{s.AllocationNodeId}.{cluster.DomainName ?? cluster.MasterNodeName}")
.ToList();
nodeNames.ForEach(f => cmdBuilder.Append($"dig +short {f};"));
string sshCommand = cmdBuilder.ToString();
_log.Info($"Get allocation nodes of task \"{taskInfo.Id}\", command \"{sshCommand}\"");
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
return(command.Result.Split('\n').Where(w => !string.IsNullOrEmpty(w))
.ToList());
}
catch (FormatException e)
{
throw new Exception($@"Exception thrown when retrieving allocation nodes used by running task (HPC job): ""{taskInfo.ScheduledJobId}"".
Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n
Command line for job submission: ""{sshCommand}""\n", e);
}
}