/// <summary>
/// Removes Job Processor workers which are IDLE for a while
/// </summary>
private IEnumerable <JobHostEntity> RemoveIdleInstancesAsNeeded(IEnumerable <JobHostEntity> overIdleJobHosts, int idleCount)
{
// does the policy state a minumum number of idle workers?
overIdleJobHosts = (idleCount > _policy.MaximumIdleJobHosts) ? overIdleJobHosts.Take(idleCount - _policy.MaximumIdleJobHosts) : overIdleJobHosts.Take(0);
string[] roleInstances = overIdleJobHosts.Select(jhe => jhe.RoleInstanceId).ToArray <string>();
if (overIdleJobHosts.Count() > 0)
{
string requestId = string.Empty;
if (RoleEnvironment.IsEmulated == false)
{
requestId = ManagementApiHelper.RemoveJobHostInstances(roleInstances);
}
if (!string.IsNullOrEmpty(requestId))
{
UpdateRoleOperationStatus(requestId);
// mark the jobHosts as deleting
foreach (var jhe in overIdleJobHosts)
{
jhe.Status = JobHostStatus.Deleting;
_jobHostRepository.UpdateJobHost(jhe);
}
}
}
return(overIdleJobHosts);
}
/// <summary>
/// Adds new instances depending on the calculated delta and if role operations are possible
/// </summary>
private bool AddNewInstancesAsRequired(int delta, bool canDoRoleOperations)
{
// Again get the number of registered Job Processors in the table
var jobHostCount = _jobHostRepository.GetJobHosts().Count();
// how many hosts 'in total' do we need?
var newCount = (delta + jobHostCount > _policy.MaximumJobHosts) ? _policy.MaximumJobHosts : (delta + jobHostCount);
string requestId = ManagementApiHelper.AddJobHostInstances(GlobalConstants.JOBTASK_PROCESS_ROLENAME, newCount);
if (!string.IsNullOrEmpty(requestId))
{
UpdateRoleOperationStatus(requestId);
canDoRoleOperations = false;
for (int i = 1; i <= newCount - jobHostCount; i++)
{
_jobHostRepository.CreateJobHost(new JobHostEntity());
}
}
return(canDoRoleOperations);
}
/// <summary>
/// Verifies if there's another role operartion going on and therefore we cannot do further role operations
/// </summary>
private bool CanDoRoleOperations()
{
// we can only add / remove instances if there is no role operation currently executing
// read the RoleOperationStatusEntity and find out if there is a outstanding requestId
// if there is then check the progress of the request.
// if the request is complete then do the next operation
bool canDoRoleOperation = true;
var roleOperation = _roleOpsRepository.GetRoleOperationStatus();
if (roleOperation != null)
{
if (string.IsNullOrEmpty(roleOperation.RequestId) == false)
{
var status = ManagementApiHelper.CheckProgress(roleOperation.RequestId);
if (!status.Equals("InProgress", StringComparison.CurrentCultureIgnoreCase))
{
// there should not be any reference to deleting (preparing items will be cleaned up when the role instances are ready)
if (status.Equals("Succeeded", StringComparison.CurrentCultureIgnoreCase))
{
var jobHosts = _jobHostRepository.GetJobHosts().Where(jh => jh.Status.Equals(JobHostStatus.Deleting)).ToList();
jobHosts.ForEach(jh => _jobHostRepository.DeleteJobHost(jh.RoleInstanceId));
}
// there should not be any reference to preparing or deleted
if (status.Equals("Failed", StringComparison.CurrentCultureIgnoreCase))
{
var prepJobHosts = _jobHostRepository.GetJobHosts().Where(jh => jh.Status.Equals(JobHostStatus.Preparing)).ToList();
prepJobHosts.ForEach(jh => _jobHostRepository.DeleteJobHost(jh.RoleInstanceId));
// put them back to idle - the delete will try again next time round
var deleteJobHosts = _jobHostRepository.GetJobHosts().Where(jh => jh.Status.Equals(JobHostStatus.Deleting)).ToList();
deleteJobHosts.ForEach(jh =>
{
jh.Status = JobHostStatus.Idle;
_jobHostRepository.UpdateJobHost(jh);
});
}
// update the RoleOperationStatus so that the requestId is empty
roleOperation.RequestId = string.Empty;
_roleOpsRepository.UpdateRoleOperationStatus(roleOperation);
}
else
{
canDoRoleOperation = false;
}
}
else
{
// create an entry
_roleOpsRepository.CreateRoleOperationStatus(new RoleOperationStatusEntity());
}
}
return(canDoRoleOperation);
}
