public ClusterNodeUsage GetCurrentClusterNodeUsage(long clusterNodeId, AdaptorUser loggedUser)
{
ClusterNodeType nodeType = GetClusterNodeTypeById(clusterNodeId);
return(SchedulerFactory.GetInstance(nodeType.Cluster.SchedulerType).CreateScheduler(nodeType.Cluster)
.GetCurrentClusterNodeUsage(nodeType));
}
/// <summary>
/// Read actual queue status from scheduler
/// </summary>
/// <param name="nodeType">Cluster node type</param>
/// <param name="responseMessage">Scheduler response message</param>
/// <returns></returns>
/// <exception cref="FormatException"></exception>
public override ClusterNodeUsage ReadQueueActualInformation(ClusterNodeType nodeType, object responseMessage)
{
string response = (string)responseMessage;
var queueInfo = new PbsProQueueInfo();
var parameters = Regex.Matches(response, @"(?<Key>[^\s].*)( = |: )(?<Value>.*)", RegexOptions.Compiled)
.Where(w => w.Success)
.Select(s => new
{
Key = s.Groups.GetValueOrDefault("Key").Value,
Value = s.Groups.GetValueOrDefault("Value").Value,
});
if (!parameters.Any())
{
throw new FormatException("Unable to parse response from PBS Professional HPC scheduler!");
}
FillingSchedulerJobResultObjectFromSchedulerAttribute(nodeType.Cluster, queueInfo, parameters.ToDictionary(i => i.Key, j => j.Value));
return(new ClusterNodeUsage
{
NodeType = nodeType,
NodesUsed = queueInfo.NodesUsed,
Priority = queueInfo.Priority,
TotalJobs = queueInfo.TotalJobs
});
}
public ClusterNodeType GetClusterNodeTypeById(long clusterNodeTypeId)
{
ClusterNodeType nodeType = unitOfWork.ClusterNodeTypeRepository.GetById(clusterNodeTypeId);
if (nodeType == null)
{
log.Error("Requested cluster node type with Id=" + clusterNodeTypeId + " does not exist in the system.");
throw new RequestedObjectDoesNotExistException("Requested cluster node type with Id=" + clusterNodeTypeId + " does not exist in the system.");
}
return(nodeType);
}
/// <summary>
/// Get actual scheduler queue status
/// </summary>
/// <param name="nodeType">Cluster node type</param>
public ClusterNodeUsage GetCurrentClusterNodeUsage(ClusterNodeType nodeType)
{
Cluster cluster = nodeType.Cluster;
ClusterAuthenticationCredentials creds = cluster.ServiceAccountCredentials;
ConnectionInfo schedulerConnection = _connectionPool.GetConnectionForUser(creds);
try
{
ClusterNodeUsage usage = _adapter.GetCurrentClusterNodeUsage(schedulerConnection.Connection, nodeType);
return(usage);
}
finally
{
_connectionPool.ReturnConnection(schedulerConnection);
}
}
public static ClusterNodeTypeExt ConvertIntToExt(this ClusterNodeType nodeType)
{
ClusterNodeTypeExt convert = new ClusterNodeTypeExt()
{
Id = nodeType.Id,
Name = nodeType.Name,
Description = nodeType.Description,
NumberOfNodes = nodeType.NumberOfNodes,
CoresPerNode = nodeType.CoresPerNode,
MaxWalltime = nodeType.MaxWalltime,
CommandTemplates = nodeType.PossibleCommands.Where(c => c.IsEnabled).Select(s => s.ConvertIntToExt())
.ToArray()
};
return(convert);
}
/// <summary>
/// Read actual queue status from scheduler
/// </summary>
/// <param name="nodeType">Cluster node type</param>
/// <param name="responseMessage">Scheduler response message</param>
/// <returns></returns>
/// <exception cref="FormatException"></exception>
public override ClusterNodeUsage ReadQueueActualInformation(ClusterNodeType nodeType, object responseMessage)
{
string response = (string)responseMessage;
string parsedNodeUsedLine = string.IsNullOrEmpty(nodeType.ClusterAllocationName)
? response
: response.Replace($"CLUSTER: {nodeType.ClusterAllocationName}\n", string.Empty);
parsedNodeUsedLine = Regex.Replace(parsedNodeUsedLine, @"[ ]|[\n]{2}", string.Empty);
if (!int.TryParse(parsedNodeUsedLine, out int nodesUsed))
{
throw new FormatException("Unable to parse cluster node usage from HPC scheduler!");
}
return(new ClusterNodeUsage
{
NodeType = nodeType,
NodesUsed = nodesUsed,
Priority = default,
public static string GetNodeSize(string clusterType, ClusterNodeType nodeType)
{
switch (nodeType)
{
case ClusterNodeType.HeadNode:
return(DefaultVmSizes.HeadNode.GetSize(clusterType));
case ClusterNodeType.WorkerNode:
return(DefaultVmSizes.WorkerNode.GetSize(clusterType));
case ClusterNodeType.ZookeeperNode:
return(DefaultVmSizes.ZookeeperNode.GetSize(clusterType));
case ClusterNodeType.EdgeNode:
return(DefaultVmSizes.EdgeNode.GetSize(clusterType));
default:
throw new ArgumentOutOfRangeException("nodeType");
}
}
/// <summary>
/// Removes instantiated objects by the network system
/// </summary>
private void ClearNetworkObjects(ClusterNodeType type)
{
switch (type)
{
case ClusterNodeType.Master:
if (Master != null)
{
Object.Destroy(Master.gameObject);
Master = null;
}
break;
case ClusterNodeType.Client:
if (Client != null)
{
Object.Destroy(Client.gameObject);
Client = null;
}
break;
}
}
public static string GetNodeSize(ClusterCreateParameters createProperties, ClusterNodeType nodeType)
{
switch (nodeType)
{
case ClusterNodeType.HeadNode:
if (!string.IsNullOrEmpty(createProperties.HeadNodeSize))
{
return(createProperties.HeadNodeSize);
}
return(DefaultVmSizes.HeadNode.GetSize(createProperties.ClusterType));
case ClusterNodeType.WorkerNode:
if (!string.IsNullOrEmpty(createProperties.WorkerNodeSize))
{
return(createProperties.WorkerNodeSize);
}
return(DefaultVmSizes.WorkerNode.GetSize(createProperties.ClusterType));
case ClusterNodeType.ZookeeperNode:
if (!string.IsNullOrEmpty(createProperties.ZookeeperNodeSize))
{
return(createProperties.ZookeeperNodeSize);
}
return(DefaultVmSizes.ZookeeperNode.GetSize(createProperties.ClusterType));
case ClusterNodeType.EdgeNode:
if (!string.IsNullOrEmpty(createProperties.EdgeNodeSize))
{
return(createProperties.EdgeNodeSize);
}
return(DefaultVmSizes.EdgeNode.GetSize(createProperties.ClusterType));
default:
throw new ArgumentOutOfRangeException("nodeType");
}
}
private static List <ScriptAction> GetScriptActionsForRoleType(Dictionary <ClusterNodeType, List <ScriptAction> > clusterScriptActions, ClusterNodeType nodeType)
{
if (clusterScriptActions == null)
{
return(null);
}
List <ScriptAction> scriptActions;
clusterScriptActions.TryGetValue(nodeType, out scriptActions);
return(scriptActions);
}
/// <summary>
/// Read actual queue status from scheduler
/// </summary>
/// <param name="nodeType">Cluster node type</param>
/// <param name="responseMessage">Scheduler response message</param>
/// <returns></returns>
/// <exception cref="FormatException"></exception>
public override ClusterNodeUsage ReadQueueActualInformation(ClusterNodeType nodeType, object responseMessage)
{
throw new NotImplementedException();
}
/// <summary>
/// Initialization method for setting up the simulation network
/// </summary>
/// <param name="type">Type of The cluster node type of this simulation</param>
/// <param name="settings">Settings used in this simulation</param>
public void Initialize(ClusterNodeType type, NetworkSettings settings)
{
Type = type;
Settings = settings;
}
private static Role GetRole(OsProfile osProfile, VirtualNetworkProfile vnetProfile, ClusterNodeType nodeType, List <ScriptAction> scriptActions, int instanceCount,
string vmSize)
{
return(new Role
{
Name = nodeType.ToString().ToLower(),
TargetInstanceCount = instanceCount,
HardwareProfile = new HardwareProfile
{
VmSize = vmSize
},
VirtualNetworkProfile = vnetProfile,
OsProfile = osProfile,
ScriptActions = scriptActions
});
}
private static List <ScriptAction> GetScriptActionsForRoleType(ClusterCreateParameters createProperties, ClusterNodeType nodeType)
{
if (createProperties.ScriptActions == null)
{
return(null);
}
List <ScriptAction> scriptActions;
createProperties.ScriptActions.TryGetValue(nodeType, out scriptActions);
return(scriptActions);
}
/// <summary>
/// Get actual scheduler queue status
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="nodeType">Cluster node type</param>
public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType)
{
SshCommandWrapper command = null;
var allocationCluster = string.Empty;
if (!string.IsNullOrEmpty(nodeType.ClusterAllocationName))
{
allocationCluster = $"--clusters={nodeType.ClusterAllocationName} ";
}
var sshCommand = $"{_commands.InterpreterCommand} 'sinfo -t alloc {allocationCluster}--partition={nodeType.Queue} -h -o \"%.6D\"'";
_log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{sshCommand}\"");
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
return(_convertor.ReadQueueActualInformation(nodeType, command.Result));
}
catch (FormatException e)
{
throw new Exception($@"Exception thrown when retrieving usage of Cluster node: ""{nodeType.Name}"".
Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n
Command line for queue usage: ""{sshCommand}""\n", e);
}
}
/// <summary>
/// Get cluster node usage
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="nodeType">ClusterNode type</param>
/// <returns></returns>
public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType)
{
var usage = new ClusterNodeUsage
{
NodeType = nodeType
};
var command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), _linuxLocalCommandScripts.CountJobsCmdPath);
_log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{command}\"");
if (int.TryParse(command.Result, out int totalJobs))
{
usage.TotalJobs = totalJobs;
}
return(usage);
}
/// <summary> /// Read queue actual information /// </summary> /// <param name="nodeType">Node type</param> /// <param name="responseMessage">Scheduler response message</param> /// <returns></returns> public abstract ClusterNodeUsage ReadQueueActualInformation(ClusterNodeType nodeType, object responseMessage);
/// <summary>
/// Get actual scheduler queue status
/// </summary>
/// <param name="connectorClient">Connector</param>
/// <param name="nodeType">Cluster node type</param>
/// <returns></returns>
/// <exception cref="Exception"></exception>
public virtual ClusterNodeUsage GetCurrentClusterNodeUsage(object connectorClient, ClusterNodeType nodeType)
{
SshCommandWrapper command = null;
string sshCommand = $"{_commands.InterpreterCommand} 'qstat -Q -f {nodeType.Queue}'";
_log.Info($"Get usage of queue \"{nodeType.Queue}\", command \"{sshCommand}\"");
try
{
command = SshCommandUtils.RunSshCommand(new SshClientAdapter((SshClient)connectorClient), sshCommand);
return(_convertor.ReadQueueActualInformation(nodeType, command.Result));
}
catch (FormatException e)
{
throw new Exception($@"Exception thrown when retrieving parameters of queue: ""{nodeType.Name}"".
Submission script result: ""{command.Result}"".\nSubmission script message: ""{command.Error}"".\n
Command line for job submission: ""{sshCommand}""\n", e);
}
}
