/// <summary>
/// Delete job directory
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
public void DeleteJobDirectory(object connectorClient, SubmittedJobInfo jobInfo)
{
string shellCommand = $"rm -Rf {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}";
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), shellCommand);
_log.Info($"Job directory \"{jobInfo.Specification.Id}\" was deleted. Result: \"{sshCommand.Result}\"");
}
protected virtual ICollection <JobFileContent> PerformSynchronizationForType(SubmittedJobInfo jobInfo, SynchronizableFiles fileType)
{
List <JobFileContent> result = new List <JobFileContent>();
if (!_fileSynchronizers.ContainsKey(fileType))
{
_fileSynchronizers[fileType] = new Dictionary <string, IFileSynchronizer>(jobInfo.Tasks.Count);
}
foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification);
FullFileSpecification fileInfo = CreateSynchronizableFileInfoForType(taskInfo.Specification, taskClusterDirectoryPath, fileType);
string sourceFilePath = FileSystemUtils.ConcatenatePaths(fileInfo.SourceDirectory, fileInfo.RelativePath);
if (!_fileSynchronizers[fileType].ContainsKey(sourceFilePath))
{
_fileSynchronizers[fileType][sourceFilePath] = CreateFileSynchronizer(fileInfo, jobInfo.Specification.ClusterUser);
}
ICollection <JobFileContent> subresult = _fileSynchronizers[fileType][sourceFilePath].SynchronizeFiles();
if (subresult != null)
{
foreach (JobFileContent content in subresult)
{
content.FileType = fileType;
content.SubmittedTaskInfoId = taskInfo.Id;
result.Add(content);
}
}
}
return(result);
}
public FileTransferMethod GetFileTransferMethod(long submittedJobInfoId, AdaptorUser loggedUser)
{
log.Info("Getting file transfer method for submitted job info ID " + submittedJobInfoId + " with user " + loggedUser.GetLogIdentification());
SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser);
var certificateGenerator = new CertificateGenerator.CertificateGenerator();
certificateGenerator.GenerateKey(2048);
string publicKey = certificateGenerator.DhiPublicKey();
string jobDir = FileSystemUtils.GetJobClusterDirectoryPath(jobInfo.Specification.FileTransferMethod.Cluster.LocalBasepath, jobInfo.Specification);
var transferMethod = new FileTransferMethod
{
Protocol = jobInfo.Specification.FileTransferMethod.Protocol,
ServerHostname = jobInfo.Specification.FileTransferMethod.ServerHostname,
SharedBasePath = jobDir,
Credentials = new AsymmetricKeyCredentials
{
Username = jobInfo.Specification.ClusterUser.Username,
PrivateKey = certificateGenerator.DhiPrivateKey(),
PublicKey = publicKey
}
};
SchedulerFactory.GetInstance(jobInfo.Specification.Cluster.SchedulerType).CreateScheduler(jobInfo.Specification.Cluster).AllowDirectFileTransferAccessForUserToJob(publicKey, jobInfo);
return(transferMethod);
}
public override void DeleteSessionFromCluster(SubmittedJobInfo jobInfo)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
UnsetReadOnlyForAllFiles(jobClusterDirectoryPath);
Directory.Delete(jobClusterDirectoryPath, true);
}
public IList <JobFileContent> DownloadPartsOfJobFilesFromCluster(long submittedJobInfoId, TaskFileOffset[] taskFileOffsets, AdaptorUser loggedUser)
{
log.Info("Getting part of job files from cluster for submitted job info ID " + submittedJobInfoId + " with user " + loggedUser.GetLogIdentification());
SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser);
IRexFileSystemManager fileManager =
FileSystemFactory.GetInstance(jobInfo.Specification.FileTransferMethod.Protocol).CreateFileSystemManager(jobInfo.Specification.FileTransferMethod);
IList <JobFileContent> result = new List <JobFileContent>();
foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks)
{
IList <TaskFileOffset> currentTaskFileOffsets = (from taskFileOffset in taskFileOffsets where taskFileOffset.SubmittedTaskInfoId == taskInfo.Id select taskFileOffset).ToList();
foreach (TaskFileOffset currentOffset in currentTaskFileOffsets)
{
ICollection <JobFileContent> contents = fileManager.DownloadPartOfJobFileFromCluster(taskInfo, currentOffset.FileType, currentOffset.Offset);
if (contents != null)
{
foreach (JobFileContent content in contents)
{
result.Add(content);
}
}
}
}
return(result);
}
/// <summary>
/// Copy job data to temp folder
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
/// <param name="hash">Hash</param>
public void CopyJobDataFromTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash)
{
string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}/.";
string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}";
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataFromTempCmdPath} {inputDirectory} {outputDirectory}");
_log.Info($"Temp data \"{hash}\" were copied to job directory \"{jobInfo.Specification.Id}\", result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Copy job data from temp folder
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
/// <param name="hash">Hash</param>
public void CopyJobDataToTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash, string path)
{
//if path is null or empty then all files and directories from ClusterLocalBasepath will be copied to hash directory
string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{path}";
inputDirectory += string.IsNullOrEmpty(path) ? "." : string.Empty;
string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}";
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataToTempCmdPath} {inputDirectory} {outputDirectory}");
_log.Info($"Job data \"{jobInfo.Specification.Id}/{path}\" were copied to temp directory \"{hash}\", result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Delete job directory
/// </summary>
/// <param name="jobInfo">Job info</param>
public void DeleteJobDirectory(SubmittedJobInfo jobInfo)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser);
try
{
_adapter.DeleteJobDirectory(schedulerConnection.Connection, jobInfo);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
/// <summary>
/// Remove direct file transfer access for user
/// </summary>
/// <param name="publicKey">Public key</param>
/// <param name="jobInfo">Job info</param>
public void RemoveDirectFileTransferAccessForUserToJob(string publicKey, SubmittedJobInfo jobInfo)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser);
try
{
_adapter.RemoveDirectFileTransferAccessForUserToJob(schedulerConnection.Connection, publicKey);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
public ICollection <FileInformation> ListChangedFilesForJob(long submittedJobInfoId, AdaptorUser loggedUser)
{
SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser);
if (jobInfo.State < JobState.Submitted || jobInfo.State == JobState.WaitingForServiceAccount)
{
return(null);
}
IRexFileSystemManager fileManager =
FileSystemFactory.GetInstance(jobInfo.Specification.FileTransferMethod.Protocol).CreateFileSystemManager(jobInfo.Specification.FileTransferMethod);
return(fileManager.ListChangedFilesForJob(jobInfo, jobInfo.SubmitTime.Value));
}
/// <summary>
/// Copy job data from temp folder
/// </summary>
/// <param name="jobInfo">Job info</param>
/// <param name="hash">Hash</param>
public void CopyJobDataFromTemp(SubmittedJobInfo jobInfo, string hash)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser);
try
{
_adapter.CopyJobDataFromTemp(schedulerConnection.Connection, jobInfo, hash);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
/// <summary>
/// Create job directory
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobInfo">Job information</param>
public void CreateJobDirectory(object connectorClient, SubmittedJobInfo jobInfo)
{
var cmdBuilder = new StringBuilder($"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id};");
foreach (var task in jobInfo.Tasks)
{
var path = !string.IsNullOrEmpty(task.Specification.ClusterTaskSubdirectory)
? $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id}/{task.Specification.ClusterTaskSubdirectory};"
: $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id};";
cmdBuilder.Append(path);
}
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), cmdBuilder.ToString());
_log.Info($"Create job directory result: \"{sshCommand.Result.Replace("\n", string.Empty)}\"");
}
public override void DeleteSessionFromCluster(SubmittedJobInfo jobInfo)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
ConnectionInfo connection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser);
try
{
string remotePath = jobClusterDirectoryPath;
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
DeleteRemoteDirectory(remotePath, client);
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
public virtual void CopyCreatedFilesFromCluster(SubmittedJobInfo jobInfo, DateTime jobSubmitTime)
{
foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification);
string[] excludedFiles =
{
taskInfo.Specification.LogFile.RelativePath,
taskInfo.Specification.ProgressFile.RelativePath,
taskInfo.Specification.StandardOutputFile,
taskInfo.Specification.StandardErrorFile
};
CopyAll(taskClusterDirectoryPath, taskInfo.Specification.LocalDirectory, true, jobSubmitTime, excludedFiles, jobInfo.Specification.ClusterUser);
}
}
public override byte[] DownloadFileFromCluster(SubmittedJobInfo jobInfo, string relativeFilePath)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
ConnectionInfo connection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
using MemoryStream stream = new MemoryStream();
string file = jobClusterDirectoryPath + relativeFilePath;
client.DownloadFile(file, stream);
return(stream.ToArray());
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
public SubmittedJobInfoExt CancelJob(long submittedJobInfoId, string sessionCode)
{
try
{
using (IUnitOfWork unitOfWork = UnitOfWorkFactory.GetUnitOfWorkFactory().CreateUnitOfWork())
{
AdaptorUser loggedUser = UserAndLimitationManagementService.GetValidatedUserForSessionCode(sessionCode, unitOfWork, UserRoleType.Submitter);
IJobManagementLogic jobLogic = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork);
SubmittedJobInfo jobInfo = jobLogic.CancelJob(submittedJobInfoId, loggedUser);
return(jobInfo.ConvertIntToExt());
}
}
catch (Exception exc)
{
ExceptionHandler.ThrowProperExternalException(exc);
return(null);
}
}
public SubmittedJobInfoExt CreateJob(JobSpecificationExt specification, string sessionCode)
{
try
{
using (IUnitOfWork unitOfWork = UnitOfWorkFactory.GetUnitOfWorkFactory().CreateUnitOfWork())
{
AdaptorUser loggedUser = UserAndLimitationManagementService.GetValidatedUserForSessionCode(sessionCode, unitOfWork, UserRoleType.Submitter);
IJobManagementLogic jobLogic = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork);
JobSpecification js = specification.ConvertExtToInt();
SubmittedJobInfo jobInfo = jobLogic.CreateJob(js, loggedUser, specification.IsExtraLong.Value);
return(jobInfo.ConvertIntToExt());
}
}
catch (Exception exc)
{
ExceptionHandler.ThrowProperExternalException(exc);
return(null);
}
}
public void EndFileTransfer(long submittedJobInfoId, FileTransferMethod transferMethod, AdaptorUser loggedUser)
{
log.Info("Removing file transfer method for submitted job info ID " + submittedJobInfoId + " with user " + loggedUser.GetLogIdentification());
SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser);
AsymmetricKeyCredentials asymmetricKeyCredentials = transferMethod.Credentials as AsymmetricKeyCredentials;
if (asymmetricKeyCredentials != null)
{
SchedulerFactory.GetInstance(jobInfo.Specification.Cluster.SchedulerType).CreateScheduler(jobInfo.Specification.Cluster).
RemoveDirectFileTransferAccessForUserToJob(asymmetricKeyCredentials.PublicKey, jobInfo);
}
else
{
log.Error("Credentials of class " + transferMethod.Credentials.GetType().Name +
" are not supported. Change the HaaSMiddleware.BusinessLogicTier.FileTransfer.FileTransferLogic.EndFileTransfer() method to add support for additional credential types.");
throw new ArgumentException("Credentials of class " + transferMethod.Credentials.GetType().Name +
" are not supported. Change the HaaSMiddleware.BusinessLogicTier.FileTransfer.FileTransferLogic.EndFileTransfer() method to add support for additional credential types.");
}
}
public static SubmittedJobInfoExt ConvertIntToExt(this SubmittedJobInfo jobInfo)
{
SubmittedJobInfoExt convert = new()
{
Id = jobInfo.Id,
Name = jobInfo.Name,
State = jobInfo.State.ConvertIntToExt(),
Project = jobInfo.Project,
CreationTime = jobInfo.CreationTime,
SubmitTime = jobInfo.SubmitTime,
StartTime = jobInfo.StartTime,
EndTime = jobInfo.EndTime,
TotalAllocatedTime = jobInfo.TotalAllocatedTime,
Tasks = jobInfo.Tasks.Select(s => s.ConvertIntToExt())
.ToArray()
};
return(convert);
}
/// <summary>
/// Convert submitted job to object for usage reporting
/// </summary>
/// <param name="job">Job</param>
/// <returns></returns>
internal static SubmittedJobInfoUsageReport ConvertToUsageReport(this SubmittedJobInfo job)
{
var jobInfoUsageReport = new SubmittedJobInfoUsageReport
{
Id = job.Id,
Name = job.Name,
State = job.State,
Project = job.Project,
CreationTime = job.CreationTime,
SubmitTime = job.SubmitTime,
StartTime = job.StartTime,
EndTime = job.EndTime,
TotalAllocatedTime = job.TotalAllocatedTime ?? 0
};
jobInfoUsageReport.TasksUsageReport = job.Tasks.Select(s => s.ConvertToUsageReport());
jobInfoUsageReport.TotalUsage = jobInfoUsageReport.TasksUsageReport.Sum(s => s.Usage);
return(jobInfoUsageReport);
}
public virtual ICollection <FileInformation> ListChangedFilesForJob(SubmittedJobInfo jobInfo, DateTime jobSubmitTime)
{
List <FileInformation> result = new List <FileInformation>();
foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification);
var changedFiles = ListChangedFilesForTask(taskClusterDirectoryPath, jobSubmitTime, jobInfo.Specification.ClusterUser);
foreach (var changedFile in changedFiles)
{
var relativeFilePath = "/" + taskInfo.Specification.Id.ToString(CultureInfo.InvariantCulture) +
Path.Combine(taskInfo.Specification.ClusterTaskSubdirectory ?? string.Empty, changedFile.FileName);
result.Add(new FileInformation
{
FileName = relativeFilePath,
LastModifiedDate = changedFile.LastModifiedDate
});
}
}
return(result);
}
public byte[] DownloadFileFromCluster(long submittedJobInfoId, string relativeFilePath, AdaptorUser loggedUser)
{
SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser);
if (jobInfo.State < JobState.Submitted || jobInfo.State == JobState.WaitingForServiceAccount)
{
return(null);
}
IRexFileSystemManager fileManager =
FileSystemFactory.GetInstance(jobInfo.Specification.FileTransferMethod.Protocol).CreateFileSystemManager(jobInfo.Specification.FileTransferMethod);
try
{
return(fileManager.DownloadFileFromCluster(jobInfo, relativeFilePath));
}
catch (SftpPathNotFoundException exception)
{
log.Warn($"{loggedUser.ToString()} is requesting not existing file '{relativeFilePath}'");
ExceptionHandler.ThrowProperExternalException(new InvalidRequestException(exception.Message));
}
return(null);
}
public abstract byte[] DownloadFileFromCluster(SubmittedJobInfo jobInfo, string relativeFilePath);
public virtual ICollection <JobFileContent> CopyLogFilesFromCluster(SubmittedJobInfo jobInfo)
{
return(PerformSynchronizationForType(jobInfo, SynchronizableFiles.LogFile));
}
public virtual void CopyInputFilesToCluster(SubmittedJobInfo jobInfo, string localJobDirectory)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
CopyAll(localJobDirectory, jobClusterDirectoryPath, false, null, null, jobInfo.Specification.ClusterUser);
}
public abstract void DeleteSessionFromCluster(SubmittedJobInfo jobInfo);
/// <summary>
/// Allow direct file transfer acces for user
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="publicKey">Public key</param>
/// <param name="jobInfo">Job information</param>
public void AllowDirectFileTransferAccessForUserToJob(object connectorClient, string publicKey, SubmittedJobInfo jobInfo)
{
publicKey = StringUtils.RemoveWhitespace(publicKey);
var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.AddFiletransferKeyCmdPath} {publicKey} {jobInfo.Specification.Id}");
_log.InfoFormat($"Allow file transfer result: \"{sshCommand.Result}\"");
}
/// <summary>
/// Convert submitted task to object for extended usage reporting
/// </summary>
/// <param name="task">Task</param>
/// <param name="job">Job</param>
/// <returns></returns>
internal static SubmittedTaskInfoExtendedUsageReport ConvertToExtendedUsageReport(this SubmittedTaskInfo task, SubmittedJobInfo job)
{
var taskInfoExtendedUsageReport = new SubmittedTaskInfoExtendedUsageReport
{
Id = task.Id,
Name = task.Name,
JobId = job.Id,
JobName = job.Name,
Project = job.Project,
Priority = task.Priority,
State = task.State,
CpuHyperThreading = task.CpuHyperThreading ?? false,
ScheduledJobId = task.ScheduledJobId,
CommandTemplateId = task.Specification.CommandTemplate.Id,
AllocatedTime = task.AllocatedTime,
StartTime = task.StartTime,
Usage = CalculateUsedResourcesForTask(task),
EndTime = task.EndTime
};
return(taskInfoExtendedUsageReport);
}
public override byte[] DownloadFileFromCluster(SubmittedJobInfo jobInfo, string relativeFilePath)
{
throw new NotImplementedException();
}
