/// <summary> /// Delete job directory /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> public void DeleteJobDirectory(object connectorClient, SubmittedJobInfo jobInfo) { string shellCommand = $"rm -Rf {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}"; var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), shellCommand); _log.Info($"Job directory \"{jobInfo.Specification.Id}\" was deleted. Result: \"{sshCommand.Result}\""); }
protected virtual ICollection <JobFileContent> PerformSynchronizationForType(SubmittedJobInfo jobInfo, SynchronizableFiles fileType) { List <JobFileContent> result = new List <JobFileContent>(); if (!_fileSynchronizers.ContainsKey(fileType)) { _fileSynchronizers[fileType] = new Dictionary <string, IFileSynchronizer>(jobInfo.Tasks.Count); } foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks) { string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification); string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification); FullFileSpecification fileInfo = CreateSynchronizableFileInfoForType(taskInfo.Specification, taskClusterDirectoryPath, fileType); string sourceFilePath = FileSystemUtils.ConcatenatePaths(fileInfo.SourceDirectory, fileInfo.RelativePath); if (!_fileSynchronizers[fileType].ContainsKey(sourceFilePath)) { _fileSynchronizers[fileType][sourceFilePath] = CreateFileSynchronizer(fileInfo, jobInfo.Specification.ClusterUser); } ICollection <JobFileContent> subresult = _fileSynchronizers[fileType][sourceFilePath].SynchronizeFiles(); if (subresult != null) { foreach (JobFileContent content in subresult) { content.FileType = fileType; content.SubmittedTaskInfoId = taskInfo.Id; result.Add(content); } } } return(result); }
public FileTransferMethod GetFileTransferMethod(long submittedJobInfoId, AdaptorUser loggedUser) { log.Info("Getting file transfer method for submitted job info ID " + submittedJobInfoId + " with user " + loggedUser.GetLogIdentification()); SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser); var certificateGenerator = new CertificateGenerator.CertificateGenerator(); certificateGenerator.GenerateKey(2048); string publicKey = certificateGenerator.DhiPublicKey(); string jobDir = FileSystemUtils.GetJobClusterDirectoryPath(jobInfo.Specification.FileTransferMethod.Cluster.LocalBasepath, jobInfo.Specification); var transferMethod = new FileTransferMethod { Protocol = jobInfo.Specification.FileTransferMethod.Protocol, ServerHostname = jobInfo.Specification.FileTransferMethod.ServerHostname, SharedBasePath = jobDir, Credentials = new AsymmetricKeyCredentials { Username = jobInfo.Specification.ClusterUser.Username, PrivateKey = certificateGenerator.DhiPrivateKey(), PublicKey = publicKey } }; SchedulerFactory.GetInstance(jobInfo.Specification.Cluster.SchedulerType).CreateScheduler(jobInfo.Specification.Cluster).AllowDirectFileTransferAccessForUserToJob(publicKey, jobInfo); return(transferMethod); }
public override void DeleteSessionFromCluster(SubmittedJobInfo jobInfo) { string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification); UnsetReadOnlyForAllFiles(jobClusterDirectoryPath); Directory.Delete(jobClusterDirectoryPath, true); }
public IList <JobFileContent> DownloadPartsOfJobFilesFromCluster(long submittedJobInfoId, TaskFileOffset[] taskFileOffsets, AdaptorUser loggedUser) { log.Info("Getting part of job files from cluster for submitted job info ID " + submittedJobInfoId + " with user " + loggedUser.GetLogIdentification()); SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser); IRexFileSystemManager fileManager = FileSystemFactory.GetInstance(jobInfo.Specification.FileTransferMethod.Protocol).CreateFileSystemManager(jobInfo.Specification.FileTransferMethod); IList <JobFileContent> result = new List <JobFileContent>(); foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks) { IList <TaskFileOffset> currentTaskFileOffsets = (from taskFileOffset in taskFileOffsets where taskFileOffset.SubmittedTaskInfoId == taskInfo.Id select taskFileOffset).ToList(); foreach (TaskFileOffset currentOffset in currentTaskFileOffsets) { ICollection <JobFileContent> contents = fileManager.DownloadPartOfJobFileFromCluster(taskInfo, currentOffset.FileType, currentOffset.Offset); if (contents != null) { foreach (JobFileContent content in contents) { result.Add(content); } } } } return(result); }
/// <summary> /// Copy job data to temp folder /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> /// <param name="hash">Hash</param> public void CopyJobDataFromTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash) { string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}/."; string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}"; var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataFromTempCmdPath} {inputDirectory} {outputDirectory}"); _log.Info($"Temp data \"{hash}\" were copied to job directory \"{jobInfo.Specification.Id}\", result: \"{sshCommand.Result}\""); }
/// <summary> /// Copy job data from temp folder /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> /// <param name="hash">Hash</param> public void CopyJobDataToTemp(object connectorClient, SubmittedJobInfo jobInfo, string hash, string path) { //if path is null or empty then all files and directories from ClusterLocalBasepath will be copied to hash directory string inputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{path}"; inputDirectory += string.IsNullOrEmpty(path) ? "." : string.Empty; string outputDirectory = $"{jobInfo.Specification.Cluster.LocalBasepath}Temp/{hash}"; var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.CopyDataToTempCmdPath} {inputDirectory} {outputDirectory}"); _log.Info($"Job data \"{jobInfo.Specification.Id}/{path}\" were copied to temp directory \"{hash}\", result: \"{sshCommand.Result}\""); }
/// <summary> /// Delete job directory /// </summary> /// <param name="jobInfo">Job info</param> public void DeleteJobDirectory(SubmittedJobInfo jobInfo) { ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser); try { _adapter.DeleteJobDirectory(schedulerConnection.Connection, jobInfo); } finally { _connectionPool.ReturnConnection(schedulerConnection); } }
/// <summary> /// Remove direct file transfer access for user /// </summary> /// <param name="publicKey">Public key</param> /// <param name="jobInfo">Job info</param> public void RemoveDirectFileTransferAccessForUserToJob(string publicKey, SubmittedJobInfo jobInfo) { ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser); try { _adapter.RemoveDirectFileTransferAccessForUserToJob(schedulerConnection.Connection, publicKey); } finally { _connectionPool.ReturnConnection(schedulerConnection); } }
public ICollection <FileInformation> ListChangedFilesForJob(long submittedJobInfoId, AdaptorUser loggedUser) { SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser); if (jobInfo.State < JobState.Submitted || jobInfo.State == JobState.WaitingForServiceAccount) { return(null); } IRexFileSystemManager fileManager = FileSystemFactory.GetInstance(jobInfo.Specification.FileTransferMethod.Protocol).CreateFileSystemManager(jobInfo.Specification.FileTransferMethod); return(fileManager.ListChangedFilesForJob(jobInfo, jobInfo.SubmitTime.Value)); }
/// <summary> /// Copy job data from temp folder /// </summary> /// <param name="jobInfo">Job info</param> /// <param name="hash">Hash</param> public void CopyJobDataFromTemp(SubmittedJobInfo jobInfo, string hash) { ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser); try { _adapter.CopyJobDataFromTemp(schedulerConnection.Connection, jobInfo, hash); } finally { _connectionPool.ReturnConnection(schedulerConnection); } }
/// <summary> /// Create job directory /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="jobInfo">Job information</param> public void CreateJobDirectory(object connectorClient, SubmittedJobInfo jobInfo) { var cmdBuilder = new StringBuilder($"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id};"); foreach (var task in jobInfo.Tasks) { var path = !string.IsNullOrEmpty(task.Specification.ClusterTaskSubdirectory) ? $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id}/{task.Specification.ClusterTaskSubdirectory};" : $"{_commandScripts.CreateJobDirectoryCmdPath} {jobInfo.Specification.Cluster.LocalBasepath}/{jobInfo.Specification.Id}/{task.Specification.Id};"; cmdBuilder.Append(path); } var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), cmdBuilder.ToString()); _log.Info($"Create job directory result: \"{sshCommand.Result.Replace("\n", string.Empty)}\""); }
public override void DeleteSessionFromCluster(SubmittedJobInfo jobInfo) { string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification); ConnectionInfo connection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser); try { string remotePath = jobClusterDirectoryPath; var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection); DeleteRemoteDirectory(remotePath, client); } finally { _connectionPool.ReturnConnection(connection); } }
public virtual void CopyCreatedFilesFromCluster(SubmittedJobInfo jobInfo, DateTime jobSubmitTime) { foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks) { string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification); string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification); string[] excludedFiles = { taskInfo.Specification.LogFile.RelativePath, taskInfo.Specification.ProgressFile.RelativePath, taskInfo.Specification.StandardOutputFile, taskInfo.Specification.StandardErrorFile }; CopyAll(taskClusterDirectoryPath, taskInfo.Specification.LocalDirectory, true, jobSubmitTime, excludedFiles, jobInfo.Specification.ClusterUser); } }
public override byte[] DownloadFileFromCluster(SubmittedJobInfo jobInfo, string relativeFilePath) { string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification); ConnectionInfo connection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser); try { var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection); using MemoryStream stream = new MemoryStream(); string file = jobClusterDirectoryPath + relativeFilePath; client.DownloadFile(file, stream); return(stream.ToArray()); } finally { _connectionPool.ReturnConnection(connection); } }
public SubmittedJobInfoExt CancelJob(long submittedJobInfoId, string sessionCode) { try { using (IUnitOfWork unitOfWork = UnitOfWorkFactory.GetUnitOfWorkFactory().CreateUnitOfWork()) { AdaptorUser loggedUser = UserAndLimitationManagementService.GetValidatedUserForSessionCode(sessionCode, unitOfWork, UserRoleType.Submitter); IJobManagementLogic jobLogic = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork); SubmittedJobInfo jobInfo = jobLogic.CancelJob(submittedJobInfoId, loggedUser); return(jobInfo.ConvertIntToExt()); } } catch (Exception exc) { ExceptionHandler.ThrowProperExternalException(exc); return(null); } }
public SubmittedJobInfoExt CreateJob(JobSpecificationExt specification, string sessionCode) { try { using (IUnitOfWork unitOfWork = UnitOfWorkFactory.GetUnitOfWorkFactory().CreateUnitOfWork()) { AdaptorUser loggedUser = UserAndLimitationManagementService.GetValidatedUserForSessionCode(sessionCode, unitOfWork, UserRoleType.Submitter); IJobManagementLogic jobLogic = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork); JobSpecification js = specification.ConvertExtToInt(); SubmittedJobInfo jobInfo = jobLogic.CreateJob(js, loggedUser, specification.IsExtraLong.Value); return(jobInfo.ConvertIntToExt()); } } catch (Exception exc) { ExceptionHandler.ThrowProperExternalException(exc); return(null); } }
public void EndFileTransfer(long submittedJobInfoId, FileTransferMethod transferMethod, AdaptorUser loggedUser) { log.Info("Removing file transfer method for submitted job info ID " + submittedJobInfoId + " with user " + loggedUser.GetLogIdentification()); SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser); AsymmetricKeyCredentials asymmetricKeyCredentials = transferMethod.Credentials as AsymmetricKeyCredentials; if (asymmetricKeyCredentials != null) { SchedulerFactory.GetInstance(jobInfo.Specification.Cluster.SchedulerType).CreateScheduler(jobInfo.Specification.Cluster). RemoveDirectFileTransferAccessForUserToJob(asymmetricKeyCredentials.PublicKey, jobInfo); } else { log.Error("Credentials of class " + transferMethod.Credentials.GetType().Name + " are not supported. Change the HaaSMiddleware.BusinessLogicTier.FileTransfer.FileTransferLogic.EndFileTransfer() method to add support for additional credential types."); throw new ArgumentException("Credentials of class " + transferMethod.Credentials.GetType().Name + " are not supported. Change the HaaSMiddleware.BusinessLogicTier.FileTransfer.FileTransferLogic.EndFileTransfer() method to add support for additional credential types."); } }
public static SubmittedJobInfoExt ConvertIntToExt(this SubmittedJobInfo jobInfo) { SubmittedJobInfoExt convert = new() { Id = jobInfo.Id, Name = jobInfo.Name, State = jobInfo.State.ConvertIntToExt(), Project = jobInfo.Project, CreationTime = jobInfo.CreationTime, SubmitTime = jobInfo.SubmitTime, StartTime = jobInfo.StartTime, EndTime = jobInfo.EndTime, TotalAllocatedTime = jobInfo.TotalAllocatedTime, Tasks = jobInfo.Tasks.Select(s => s.ConvertIntToExt()) .ToArray() }; return(convert); }
/// <summary> /// Convert submitted job to object for usage reporting /// </summary> /// <param name="job">Job</param> /// <returns></returns> internal static SubmittedJobInfoUsageReport ConvertToUsageReport(this SubmittedJobInfo job) { var jobInfoUsageReport = new SubmittedJobInfoUsageReport { Id = job.Id, Name = job.Name, State = job.State, Project = job.Project, CreationTime = job.CreationTime, SubmitTime = job.SubmitTime, StartTime = job.StartTime, EndTime = job.EndTime, TotalAllocatedTime = job.TotalAllocatedTime ?? 0 }; jobInfoUsageReport.TasksUsageReport = job.Tasks.Select(s => s.ConvertToUsageReport()); jobInfoUsageReport.TotalUsage = jobInfoUsageReport.TasksUsageReport.Sum(s => s.Usage); return(jobInfoUsageReport); }
public virtual ICollection <FileInformation> ListChangedFilesForJob(SubmittedJobInfo jobInfo, DateTime jobSubmitTime) { List <FileInformation> result = new List <FileInformation>(); foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks) { string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification); string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification); var changedFiles = ListChangedFilesForTask(taskClusterDirectoryPath, jobSubmitTime, jobInfo.Specification.ClusterUser); foreach (var changedFile in changedFiles) { var relativeFilePath = "/" + taskInfo.Specification.Id.ToString(CultureInfo.InvariantCulture) + Path.Combine(taskInfo.Specification.ClusterTaskSubdirectory ?? string.Empty, changedFile.FileName); result.Add(new FileInformation { FileName = relativeFilePath, LastModifiedDate = changedFile.LastModifiedDate }); } } return(result); }
public byte[] DownloadFileFromCluster(long submittedJobInfoId, string relativeFilePath, AdaptorUser loggedUser) { SubmittedJobInfo jobInfo = LogicFactory.GetLogicFactory().CreateJobManagementLogic(unitOfWork).GetSubmittedJobInfoById(submittedJobInfoId, loggedUser); if (jobInfo.State < JobState.Submitted || jobInfo.State == JobState.WaitingForServiceAccount) { return(null); } IRexFileSystemManager fileManager = FileSystemFactory.GetInstance(jobInfo.Specification.FileTransferMethod.Protocol).CreateFileSystemManager(jobInfo.Specification.FileTransferMethod); try { return(fileManager.DownloadFileFromCluster(jobInfo, relativeFilePath)); } catch (SftpPathNotFoundException exception) { log.Warn($"{loggedUser.ToString()} is requesting not existing file '{relativeFilePath}'"); ExceptionHandler.ThrowProperExternalException(new InvalidRequestException(exception.Message)); } return(null); }
public abstract byte[] DownloadFileFromCluster(SubmittedJobInfo jobInfo, string relativeFilePath);
public virtual ICollection <JobFileContent> CopyLogFilesFromCluster(SubmittedJobInfo jobInfo) { return(PerformSynchronizationForType(jobInfo, SynchronizableFiles.LogFile)); }
public virtual void CopyInputFilesToCluster(SubmittedJobInfo jobInfo, string localJobDirectory) { string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification); CopyAll(localJobDirectory, jobClusterDirectoryPath, false, null, null, jobInfo.Specification.ClusterUser); }
public abstract void DeleteSessionFromCluster(SubmittedJobInfo jobInfo);
/// <summary> /// Allow direct file transfer acces for user /// </summary> /// <param name="connectorClient">Connector</param> /// <param name="publicKey">Public key</param> /// <param name="jobInfo">Job information</param> public void AllowDirectFileTransferAccessForUserToJob(object connectorClient, string publicKey, SubmittedJobInfo jobInfo) { publicKey = StringUtils.RemoveWhitespace(publicKey); var sshCommand = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.AddFiletransferKeyCmdPath} {publicKey} {jobInfo.Specification.Id}"); _log.InfoFormat($"Allow file transfer result: \"{sshCommand.Result}\""); }
/// <summary> /// Convert submitted task to object for extended usage reporting /// </summary> /// <param name="task">Task</param> /// <param name="job">Job</param> /// <returns></returns> internal static SubmittedTaskInfoExtendedUsageReport ConvertToExtendedUsageReport(this SubmittedTaskInfo task, SubmittedJobInfo job) { var taskInfoExtendedUsageReport = new SubmittedTaskInfoExtendedUsageReport { Id = task.Id, Name = task.Name, JobId = job.Id, JobName = job.Name, Project = job.Project, Priority = task.Priority, State = task.State, CpuHyperThreading = task.CpuHyperThreading ?? false, ScheduledJobId = task.ScheduledJobId, CommandTemplateId = task.Specification.CommandTemplate.Id, AllocatedTime = task.AllocatedTime, StartTime = task.StartTime, Usage = CalculateUsedResourcesForTask(task), EndTime = task.EndTime }; return(taskInfoExtendedUsageReport); }
public override byte[] DownloadFileFromCluster(SubmittedJobInfo jobInfo, string relativeFilePath) { throw new NotImplementedException(); }