public bool IsUserAvailableToRun(ClusterAuthenticationCredentials user)
{
//List all unfinished jobs and
IEnumerable <SubmittedJobInfo> allRunningJobs = unitOfWork.SubmittedJobInfoRepository.GetAllUnfinished();
List <SubmittedJobInfo> userRunningJobs = allRunningJobs.Where(w => w.Specification.ClusterUser == user && (w.State == JobState.Running ||
w.State == JobState.Queued ||
w.State == JobState.Submitted))
.ToList();
return(userRunningJobs.Any() ? false : true);
}
private ConnectionInfo ExpandPoolAndGetConnection(ClusterAuthenticationCredentials cred)
{
ConnectionInfo connection = InitializeConnection(cred);
actualSize++;
if (poolCleanTimer != null && actualSize > minSize)
{
poolCleanTimer.Start();
}
log.DebugFormat("Connection pool expanded with acc {0} - actual size {1}", cred.Username, actualSize);
return(connection);
}
private ConnectionInfo InitializeConnection(ClusterAuthenticationCredentials cred)
{
object connectionObject = adapter.CreateConnectionObject(_masterNodeName, _remoteTimeZone, cred, _port);
ConnectionInfo connection = new ConnectionInfo
{
Connection = connectionObject,
LastUsed = DateTime.UtcNow,
AuthCredentials = cred
};
adapter.Connect(connection.Connection, _masterNodeName, cred);
return(connection);
}
/// <summary>
/// Get generic command templates parameters from script
/// </summary>
/// <param name="cluster">Cluster</param>
/// <param name="userScriptPath">Generic script path</param>
/// <returns></returns>
public IEnumerable <string> GetParametersFromGenericUserScript(Cluster cluster, string userScriptPath)
{
ClusterAuthenticationCredentials creds = cluster.ServiceAccountCredentials;
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(creds);
try
{
return(_adapter.GetParametersFromGenericUserScript(schedulerConnection.Connection, userScriptPath));
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
/// <summary>
/// Get actual scheduler queue status
/// </summary>
/// <param name="nodeType">Cluster node type</param>
public ClusterNodeUsage GetCurrentClusterNodeUsage(ClusterNodeType nodeType)
{
Cluster cluster = nodeType.Cluster;
ClusterAuthenticationCredentials creds = cluster.ServiceAccountCredentials;
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(creds);
try
{
ClusterNodeUsage usage = _adapter.GetCurrentClusterNodeUsage(schedulerConnection.Connection, nodeType);
return(usage);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
public ConnectionInfo GetConnectionForUser(ClusterAuthenticationCredentials credentials)
{
//log.DebugFormat("Thread {0} Requesting connectin for {1}",Thread.CurrentThread.ManagedThreadId, credentials.Username);
ConnectionInfo connection = null;
do
{
lock (pool) {
if (pool.ContainsKey(credentials.Id) && pool[credentials.Id].Last != null)
{
// Free connection found
connection = pool[credentials.Id].Last.Value;
// Remove it from cache
pool[credentials.Id].RemoveLast();
}
else
{
if (actualSize < maxSize)
{
// If there is space free, create new connection
connection = ExpandPoolAndGetConnection(credentials);
}
else if (HasAnyFreeConnection())
{
// If pool is full, drop oldest connection
// Find oldest connection in pool
ConnectionInfo oldest = FindOldestConnection();
// Drop it
RemoveConnectionFromPool(oldest);
// Expand pool with newly created one
connection = ExpandPoolAndGetConnection(credentials);
}
else
{
// Wait for any returned connection
Monitor.Wait(pool);
}
}
}
//log.InfoFormat("Thread {0} Got connection for {1}", Thread.CurrentThread.ManagedThreadId, credentials.Username);
} while (connection == null);
return(connection);
}
public ClusterAuthenticationCredentials GetNextAvailableUserCredentials(long clusterId)
{
// Get credentials for cluster
Cluster cluster = unitOfWork.ClusterRepository.GetById(clusterId);
if (cluster == null)
{
log.Error("Requested cluster with Id=" + clusterId + " does not exist in the system.");
throw new RequestedObjectDoesNotExistException("Requested cluster with Id=" + clusterId + " does not exist in the system.");
}
List <ClusterAuthenticationCredentials> credentials =
(from account in cluster.AuthenticationCredentials where account != cluster.ServiceAccountCredentials orderby account.Id ascending select account).ToList();
long lastUsedId = ClusterUserCache.GetLastUserId(cluster);
if (lastUsedId == 0)
{ // No user has been used from this cluster
// return first usable account
ClusterUserCache.SetLastUserId(cluster, credentials[0].Id);
log.DebugFormat("Using initial cluster account: {0}", credentials[0].Username);
return(credentials[0]);
}
else
{
// Return first user with Id higher than the last one
ClusterAuthenticationCredentials creds = (from account in credentials where account.Id > lastUsedId select account).FirstOrDefault();
if (creds == null)
{
// No credentials with Id higher than last used found
// use first usable account
creds = credentials[0];
}
ClusterUserCache.SetLastUserId(cluster, creds.Id);
log.DebugFormat("Using cluster account: {0}", creds.Username);
return(creds);
}
}
protected override IFileSynchronizer CreateFileSynchronizer(FullFileSpecification fileInfo, ClusterAuthenticationCredentials credentials)
{
SftpFullNameSynchronizer synchronizer = (SftpFullNameSynchronizer)_synchronizerFactory.CreateFileSynchronizer(fileInfo, credentials);
synchronizer.ConnectionPool = _connectionPool;
return(synchronizer);
}
protected abstract IFileSynchronizer CreateFileSynchronizer(FullFileSpecification fileInfo, ClusterAuthenticationCredentials credentials);
private static ClusterAuthenticationCredentialsAuthType GetCredentialsAuthenticationType(ClusterAuthenticationCredentials credential)
{
if (!string.IsNullOrEmpty(credential.Password) && !string.IsNullOrEmpty(credential.PrivateKeyFile))
{
return(ClusterAuthenticationCredentialsAuthType.PasswordAndPrivateKey);
}
if (!string.IsNullOrEmpty(credential.PrivateKeyFile))
{
return(ClusterAuthenticationCredentialsAuthType.PrivateKey);
}
if (!string.IsNullOrEmpty(credential.Password))
{
switch (credential.Cluster.ConnectionProtocol)
{
case ClusterConnectionProtocol.MicrosoftHpcApi:
return(ClusterAuthenticationCredentialsAuthType.Password);
case ClusterConnectionProtocol.Ssh:
return(ClusterAuthenticationCredentialsAuthType.Password);
case ClusterConnectionProtocol.SshInteractive:
return(ClusterAuthenticationCredentialsAuthType.PasswordInteractive);
default:
return(ClusterAuthenticationCredentialsAuthType.Password);
}
}
return(ClusterAuthenticationCredentialsAuthType.PrivateKeyInSshAgent);
}
internal override IFileSynchronizer CreateFileSynchronizer(FullFileSpecification syncFile, ClusterAuthenticationCredentials credentials)
{
return(syncFile.NameSpecification switch
{
FileNameSpecification.FullName => new NetworkShareFullNameSynchronizer(syncFile),
_ => default,
/// <summary>
/// Create ssh connection object
/// </summary>
/// <param name="masterNodeName">Master node name</param>
/// <param name="credentials">Credentials</param>
/// <param name="port">Port</param>
/// <returns></returns>
public object CreateConnectionObject(string masterNodeName, string remoteTimeZone, ClusterAuthenticationCredentials credentials, int?port)
{
return(credentials.AuthenticationType switch
{
ClusterAuthenticationCredentialsAuthType.Password => CreateConnectionObjectUsingPasswordAuthentication(masterNodeName, credentials.Username, credentials.Password, port),
ClusterAuthenticationCredentialsAuthType.PasswordInteractive => CreateConnectionObjectUsingPasswordAuthenticationWithKeyboardInteractive(masterNodeName, credentials.Username, credentials.Password),
ClusterAuthenticationCredentialsAuthType.PasswordAndPrivateKey => CreateConnectionObjectUsingPrivateKeyAndPasswordAuthentication(masterNodeName, credentials.Username, credentials.Password, credentials.PrivateKeyFile, credentials.PrivateKeyPassword, port),
ClusterAuthenticationCredentialsAuthType.PrivateKey => CreateConnectionObjectUsingPrivateKeyAuthentication(masterNodeName, credentials.Username, credentials.PrivateKeyFile, credentials.PrivateKeyPassword, port),
ClusterAuthenticationCredentialsAuthType.PrivateKeyInSshAgent => CreateConnectionObjectUsingNoAuthentication(masterNodeName, credentials.Username),
_ => throw new NotImplementedException("Cluster authentication credentials authentication type is not allowed!")
});
internal abstract IFileSynchronizer CreateFileSynchronizer(FullFileSpecification syncFile, ClusterAuthenticationCredentials credentials);
/// <summary>
/// Submit job to scheduler
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobSpecification">Job specification</param>
/// <param name="credentials">Credentials</param>
/// <returns></returns>
public virtual IEnumerable <SubmittedTaskInfo> SubmitJob(object connectorClient, JobSpecification jobSpecification, ClusterAuthenticationCredentials credentials)
{
var shellCommandSb = new StringBuilder();
SshCommandWrapper command = null;
string shellCommand = (string)_convertor.ConvertJobSpecificationToJob(jobSpecification, null);
_log.Info($"Submitting job \"{jobSpecification.Id}\", command \"{shellCommand}\"");
string sshCommandBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(shellCommand));
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.ExecutieCmdPath} {sshCommandBase64}");
shellCommandSb.Clear();
//compose command with parameters of job and task IDs
shellCommandSb.Append($"{_linuxLocalCommandScripts.RunLocalCmdPath} {jobSpecification.FileTransferMethod.Cluster.LocalBasepath}/{jobSpecification.Id}/");
jobSpecification.Tasks.ForEach(task => shellCommandSb.Append($" {task.Id}"));
//log local HPC Run script to log file
shellCommandSb.Append($" >> {jobSpecification.FileTransferMethod.Cluster.LocalBasepath}/{jobSpecification.Id}/job_log.txt &");
shellCommand = shellCommandSb.ToString();
sshCommandBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(shellCommand));
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), $"{_commandScripts.ExecutieCmdPath} {sshCommandBase64}");
return(GetActualTasksInfo(connectorClient, jobSpecification.Cluster, new string[] { $"{jobSpecification.Id}" }));
}
protected abstract ICollection <FileInformation> ListChangedFilesForTask(string taskClusterDirectoryPath, DateTime?jobSubmitTime, ClusterAuthenticationCredentials clusterAuthenticationCredentials);
/// <summary>
/// Get actual tasks
/// </summary>
/// <param name="submitedTasksInfo">Submitted tasks ids</param>
/// <param name="credentials">Credentials</param>
/// <returns></returns>
public IEnumerable <SubmittedTaskInfo> GetActualTasksInfo(IEnumerable <SubmittedTaskInfo> submitedTasksInfo, ClusterAuthenticationCredentials credentials)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(credentials);
try
{
var tasks = _adapter.GetActualTasksInfo(schedulerConnection.Connection, credentials.Cluster, submitedTasksInfo);
return(tasks);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
protected override void CopyAll(string source, string target, bool overwrite, DateTime?lastModificationLimit,
string[] excludedFiles, ClusterAuthenticationCredentials credentials)
{
FileSystemUtils.CopyAll(source, target, overwrite, lastModificationLimit, excludedFiles);
}
public SftpFullNameSynchronizer(FullFileSpecification syncFile, ClusterAuthenticationCredentials credentials)
{
_credentials = credentials;
SyncFileInfo = syncFile;
Offset = 0;
}
protected override ICollection <FileInformation> ListChangedFilesForTask(string taskClusterDirectoryPath, DateTime?lastModificationLimit, ClusterAuthenticationCredentials credentials)
{
ConnectionInfo connection = _connectionPool.GetConnectionForUser(credentials);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
return(ListChangedFilesInDirectory(taskClusterDirectoryPath, taskClusterDirectoryPath, lastModificationLimit, credentials, client));
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
protected override ICollection <FileInformation> ListChangedFilesForTask(string taskClusterDirectoryPath, DateTime?jobSubmitTime, ClusterAuthenticationCredentials clusterAuthenticationCredentials)
{
throw new NotImplementedException();
}
/// <summary>
/// Submit job to scheduler
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="jobSpecification">Job specification</param>
/// <param name="credentials">Credentials</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public virtual IEnumerable <SubmittedTaskInfo> SubmitJob(object connectorClient, JobSpecification jobSpecification, ClusterAuthenticationCredentials credentials)
{
var jobIdsWithJobArrayIndexes = new List <string>();
SshCommandWrapper command = null;
string sshCommand = (string)_convertor.ConvertJobSpecificationToJob(jobSpecification, "qsub -koed");
_log.Info($"Submitting job \"{jobSpecification.Id}\", command \"{sshCommand}\"");
string sshCommandBase64 = $"{_commands.InterpreterCommand} '{_commands.ExecutieCmdScriptPath} {Convert.ToBase64String(Encoding.UTF8.GetBytes(sshCommand))}'";
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommandBase64);
var jobIds = _convertor.GetJobIds(command.Result).ToList();
for (int i = 0; i < jobSpecification.Tasks.Count; i++)
{
jobIdsWithJobArrayIndexes.AddRange(string.IsNullOrEmpty(jobSpecification.Tasks[i].JobArrays)
? new List <string> {
jobIds[i]
}
: CombineScheduledJobIdWithJobArrayIndexes(jobIds[i], jobSpecification.Tasks[i].JobArrays));
}
return(GetActualTasksInfo(connectorClient, jobSpecification.Cluster, jobIdsWithJobArrayIndexes));
}
catch (FormatException e)
{
throw new Exception(@$ "Exception thrown when submitting a job: " "{jobSpecification.Name}" " to the cluster: " "{jobSpecification.Cluster.Name}" ".
Submission script result: " "{command.Result}" ".\nSubmission script error message: " "{command.Error}" ".\n
Command line for job submission: " "{sshCommandBase64}" ".\n", e);
}
}
protected override IFileSynchronizer CreateFileSynchronizer(FullFileSpecification fileInfo, ClusterAuthenticationCredentials credentials)
{
return(_synchronizerFactory.CreateFileSynchronizer(fileInfo, credentials));
}
private ICollection <FileInformation> ListChangedFilesInDirectory(string rootDirectory, string currentDirectory, DateTime?lastModificationLimit, ClusterAuthenticationCredentials credentials, SftpClientAdapter client)
{
List <FileInformation> results = new List <FileInformation>();
foreach (SftpFile file in client.ListDirectory(currentDirectory))
{
if (file.Name == "." || file.Name == "..")
{
continue;
}
if (file.IsDirectory)
{
results.AddRange(ListChangedFilesInDirectory(rootDirectory, FileSystemUtils.ConcatenatePaths(currentDirectory, file.Name), lastModificationLimit, credentials, client));
}
else if (((!lastModificationLimit.HasValue) || (lastModificationLimit.Value <= file.LastWriteTime)))
{
results.Add(new FileInformation
{
FileName = file.FullName.Replace(rootDirectory, string.Empty),
LastModifiedDate = file.LastWriteTime
});
}
}
return(results);
}
protected override void CopyAll(string source, string target, bool overwrite, DateTime?lastModificationLimit, string[] excludedFiles, ClusterAuthenticationCredentials credentials)
{
ConnectionInfo connection = _connectionPool.GetConnectionForUser(credentials);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
if (Uri.IsWellFormedUriString(target, UriKind.Absolute))
{
CopyAllToSftp(source, target, overwrite, lastModificationLimit, client, excludedFiles);
}
else
{
if (Uri.IsWellFormedUriString(source, UriKind.Absolute))
{
CopyAllFromSftp(source, target, overwrite, lastModificationLimit, client, excludedFiles);
}
else
{
FileSystemUtils.CopyAll(source, target, overwrite, lastModificationLimit, excludedFiles);
}
}
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
/// <summary>
/// Submit job to scheduler
/// </summary>
/// <param name="jobSpecification">Job specification</param>
/// <param name="credentials">Credentials</param>
/// <returns></returns>
public IEnumerable <SubmittedTaskInfo> SubmitJob(JobSpecification jobSpecification, ClusterAuthenticationCredentials credentials)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(credentials);
try
{
var tasks = _adapter.SubmitJob(schedulerConnection.Connection, jobSpecification, credentials);
return(tasks);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
public object CreateConnectionObject(string masterNodeName, string remoteTimeZone, ClusterAuthenticationCredentials credentials, int?port = null)
{
if (!string.IsNullOrEmpty(credentials.PrivateKeyFile))
{
return(CreateConnectionObjectUsingPrivateKeyAuthentication(masterNodeName, remoteTimeZone, credentials.Username, credentials.PrivateKeyFile, credentials.PrivateKeyPassword, port));
}
else
{
if (!string.IsNullOrEmpty(credentials.Password))
{
return(credentials.Cluster.ConnectionProtocol switch
{
ClusterConnectionProtocol.MicrosoftHpcApi => CreateConnectionObjectUsingPasswordAuthentication(masterNodeName, remoteTimeZone, credentials.Username, credentials.Password, port),
ClusterConnectionProtocol.Ssh => CreateConnectionObjectUsingPasswordAuthentication(masterNodeName, remoteTimeZone, credentials.Username, credentials.Password, port),
ClusterConnectionProtocol.SshInteractive => CreateConnectionObjectUsingPasswordAuthenticationWithKeyboardInteractive(masterNodeName, remoteTimeZone, credentials.Username, credentials.Password),
_ => CreateConnectionObjectUsingPasswordAuthentication(masterNodeName, remoteTimeZone, credentials.Username, credentials.Password),
});
/// <summary>
/// Cancel job
/// </summary>
/// <param name="submitedTasksInfo">Submitted tasks id´s</param>
/// <param name="message">Message</param>
/// <param name="credentials">Credentials</param>
public void CancelJob(IEnumerable <SubmittedTaskInfo> submitedTasksInfo, string message, ClusterAuthenticationCredentials credentials)
{
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(credentials);
try
{
_adapter.CancelJob(schedulerConnection.Connection, submitedTasksInfo, message);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
protected abstract void CopyAll(string source, string target, bool overwrite, DateTime?lastModificationLimit, string[] excludedFiles, ClusterAuthenticationCredentials credentials);
