private async Task HandleRequest(ScaleUpRequestEntity request)
{
try
{
switch (await PerformRequest(request))
{
case RequestStatus.Completed:
try
{
await _requestStore.Delete(request);
}
catch (StorageException ex)
when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.PreconditionFailed)
{
// ETag mismatch - request was updated.
// We will pick it up again on the next loop.
}
break;
case RequestStatus.InProgress:
// Nothing to do.
break;
}
}
catch (Exception ex) when(!(ex is TaskCanceledException))
{
_logger.LogError(ex, "Unexpected error when handling scale request '{0}'", request.ETag);
}
}
private static (int lowPriority, int dedicated) CalculateNodeTargets(ScaleUpRequestEntity request, Pool pool) => NodeTargetsCalculator.Calculemus(
private async Task <RequestStatus> PerformRequest(ScaleUpRequestEntity request)
{
var env = await _envs.GetEnvironment(request.EnvironmentName);
if (env == null)
{
_logger.LogInformation(
"Environment '{0}' has been deleted, discarding scale request '{1}'",
request.EnvironmentName,
request.ETag);
return(RequestStatus.Completed);
}
using (var batchClient = await _clientProvider.CreateBatchManagementClient(env.SubscriptionId))
{
try
{
var pool = await batchClient.Pool.GetAsync(env.BatchAccount.ResourceGroupName, env.BatchAccount.Name, request.PoolName);
if (pool == null || pool.ProvisioningState == PoolProvisioningState.Deleting)
{
_logger.LogInformation(
"Pool '{0}' (in environment '{1}') has been deleted, discarding scale request '{2}'",
request.PoolName,
request.EnvironmentName,
request.ETag);
return(RequestStatus.Completed);
}
if (pool.AllocationState == AllocationState.Resizing)
{
var op = pool.ResizeOperationStatus;
if (op != null && ((op.TargetDedicatedNodes ?? 0) + (op.TargetLowPriorityNodes ?? 0)) >= request.TargetNodes)
{
_logger.LogInformation(
"A resize operation on pool '{0}' (in environment '{1}') has made scale request '{2}' redundant, discarding it",
request.PoolName,
request.EnvironmentName,
request.ETag);
return(RequestStatus.Completed);
}
else
{
_logger.LogInformation(
"Pool '{0}' (in environment '{1}') is already being resized. Waiting to apply scale request '{2}'",
request.PoolName,
request.EnvironmentName,
request.ETag);
return(RequestStatus.InProgress);
}
}
var targets = CalculateNodeTargets(request, pool);
var newPool =
new Pool(name: pool.Name)
{
ScaleSettings =
new ScaleSettings
{
FixedScale =
new FixedScaleSettings(
targetLowPriorityNodes: targets.lowPriority,
targetDedicatedNodes: targets.dedicated)
}
};
await batchClient.Pool.UpdateAsync(env.BatchAccount.ResourceGroupName, env.BatchAccount.Name, request.PoolName, newPool);
_logger.LogInformation(
"Successfully applied scale request '{0}' to pool '{1}' (in environment '{2}')",
request.ETag,
request.PoolName,
request.EnvironmentName);
}
catch (CloudException ce) when(ce.ResourceNotFound())
{
// Pool is gone - complete the request to remove it.
}
return(RequestStatus.Completed);
}
}
