private async Task HandleRequest(ScaleUpRequestEntity request) { try { switch (await PerformRequest(request)) { case RequestStatus.Completed: try { await _requestStore.Delete(request); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed) { // ETag mismatch - request was updated. // We will pick it up again on the next loop. } break; case RequestStatus.InProgress: // Nothing to do. break; } } catch (Exception ex) when(!(ex is TaskCanceledException)) { _logger.LogError(ex, "Unexpected error when handling scale request '{0}'", request.ETag); } }
private static (int lowPriority, int dedicated) CalculateNodeTargets(ScaleUpRequestEntity request, Pool pool) => NodeTargetsCalculator.Calculemus(
private async Task <RequestStatus> PerformRequest(ScaleUpRequestEntity request) { var env = await _envs.GetEnvironment(request.EnvironmentName); if (env == null) { _logger.LogInformation( "Environment '{0}' has been deleted, discarding scale request '{1}'", request.EnvironmentName, request.ETag); return(RequestStatus.Completed); } using (var batchClient = await _clientProvider.CreateBatchManagementClient(env.SubscriptionId)) { try { var pool = await batchClient.Pool.GetAsync(env.BatchAccount.ResourceGroupName, env.BatchAccount.Name, request.PoolName); if (pool == null || pool.ProvisioningState == PoolProvisioningState.Deleting) { _logger.LogInformation( "Pool '{0}' (in environment '{1}') has been deleted, discarding scale request '{2}'", request.PoolName, request.EnvironmentName, request.ETag); return(RequestStatus.Completed); } if (pool.AllocationState == AllocationState.Resizing) { var op = pool.ResizeOperationStatus; if (op != null && ((op.TargetDedicatedNodes ?? 0) + (op.TargetLowPriorityNodes ?? 0)) >= request.TargetNodes) { _logger.LogInformation( "A resize operation on pool '{0}' (in environment '{1}') has made scale request '{2}' redundant, discarding it", request.PoolName, request.EnvironmentName, request.ETag); return(RequestStatus.Completed); } else { _logger.LogInformation( "Pool '{0}' (in environment '{1}') is already being resized. Waiting to apply scale request '{2}'", request.PoolName, request.EnvironmentName, request.ETag); return(RequestStatus.InProgress); } } var targets = CalculateNodeTargets(request, pool); var newPool = new Pool(name: pool.Name) { ScaleSettings = new ScaleSettings { FixedScale = new FixedScaleSettings( targetLowPriorityNodes: targets.lowPriority, targetDedicatedNodes: targets.dedicated) } }; await batchClient.Pool.UpdateAsync(env.BatchAccount.ResourceGroupName, env.BatchAccount.Name, request.PoolName, newPool); _logger.LogInformation( "Successfully applied scale request '{0}' to pool '{1}' (in environment '{2}')", request.ETag, request.PoolName, request.EnvironmentName); } catch (CloudException ce) when(ce.ResourceNotFound()) { // Pool is gone - complete the request to remove it. } return(RequestStatus.Completed); } }