/// <summary> /// Removes Job Processor workers which are IDLE for a while /// </summary> private IEnumerable <JobHostEntity> RemoveIdleInstancesAsNeeded(IEnumerable <JobHostEntity> overIdleJobHosts, int idleCount) { // does the policy state a minumum number of idle workers? overIdleJobHosts = (idleCount > _policy.MaximumIdleJobHosts) ? overIdleJobHosts.Take(idleCount - _policy.MaximumIdleJobHosts) : overIdleJobHosts.Take(0); string[] roleInstances = overIdleJobHosts.Select(jhe => jhe.RoleInstanceId).ToArray <string>(); if (overIdleJobHosts.Count() > 0) { string requestId = string.Empty; if (RoleEnvironment.IsEmulated == false) { requestId = ManagementApiHelper.RemoveJobHostInstances(roleInstances); } if (!string.IsNullOrEmpty(requestId)) { UpdateRoleOperationStatus(requestId); // mark the jobHosts as deleting foreach (var jhe in overIdleJobHosts) { jhe.Status = JobHostStatus.Deleting; _jobHostRepository.UpdateJobHost(jhe); } } } return(overIdleJobHosts); }
/// <summary> /// Adds new instances depending on the calculated delta and if role operations are possible /// </summary> private bool AddNewInstancesAsRequired(int delta, bool canDoRoleOperations) { // Again get the number of registered Job Processors in the table var jobHostCount = _jobHostRepository.GetJobHosts().Count(); // how many hosts 'in total' do we need? var newCount = (delta + jobHostCount > _policy.MaximumJobHosts) ? _policy.MaximumJobHosts : (delta + jobHostCount); string requestId = ManagementApiHelper.AddJobHostInstances(GlobalConstants.JOBTASK_PROCESS_ROLENAME, newCount); if (!string.IsNullOrEmpty(requestId)) { UpdateRoleOperationStatus(requestId); canDoRoleOperations = false; for (int i = 1; i <= newCount - jobHostCount; i++) { _jobHostRepository.CreateJobHost(new JobHostEntity()); } } return(canDoRoleOperations); }
/// <summary> /// Verifies if there's another role operartion going on and therefore we cannot do further role operations /// </summary> private bool CanDoRoleOperations() { // we can only add / remove instances if there is no role operation currently executing // read the RoleOperationStatusEntity and find out if there is a outstanding requestId // if there is then check the progress of the request. // if the request is complete then do the next operation bool canDoRoleOperation = true; var roleOperation = _roleOpsRepository.GetRoleOperationStatus(); if (roleOperation != null) { if (string.IsNullOrEmpty(roleOperation.RequestId) == false) { var status = ManagementApiHelper.CheckProgress(roleOperation.RequestId); if (!status.Equals("InProgress", StringComparison.CurrentCultureIgnoreCase)) { // there should not be any reference to deleting (preparing items will be cleaned up when the role instances are ready) if (status.Equals("Succeeded", StringComparison.CurrentCultureIgnoreCase)) { var jobHosts = _jobHostRepository.GetJobHosts().Where(jh => jh.Status.Equals(JobHostStatus.Deleting)).ToList(); jobHosts.ForEach(jh => _jobHostRepository.DeleteJobHost(jh.RoleInstanceId)); } // there should not be any reference to preparing or deleted if (status.Equals("Failed", StringComparison.CurrentCultureIgnoreCase)) { var prepJobHosts = _jobHostRepository.GetJobHosts().Where(jh => jh.Status.Equals(JobHostStatus.Preparing)).ToList(); prepJobHosts.ForEach(jh => _jobHostRepository.DeleteJobHost(jh.RoleInstanceId)); // put them back to idle - the delete will try again next time round var deleteJobHosts = _jobHostRepository.GetJobHosts().Where(jh => jh.Status.Equals(JobHostStatus.Deleting)).ToList(); deleteJobHosts.ForEach(jh => { jh.Status = JobHostStatus.Idle; _jobHostRepository.UpdateJobHost(jh); }); } // update the RoleOperationStatus so that the requestId is empty roleOperation.RequestId = string.Empty; _roleOpsRepository.UpdateRoleOperationStatus(roleOperation); } else { canDoRoleOperation = false; } } else { // create an entry _roleOpsRepository.CreateRoleOperationStatus(new RoleOperationStatusEntity()); } } return(canDoRoleOperation); }