public async Task <ActionResult <PoolListModel> > Index(string envId)
{
var environment = await _environmentCoordinator.GetEnvironment(envId);
if (environment == null)
{
return(RedirectToAction("Index", "Environments"));
}
var model = new PoolListModel
{
EnvironmentName = envId,
BatchAccount = environment.BatchAccount.Name,
Location = environment.BatchAccount.Location
};
try
{
var pools = (await _poolCoordinator.ListPools(environment)).Select(pool => new PoolListDetailsModel(pool));
model.Pools.AddRange(pools);
}
catch (Exception ex)
{
ModelState.AddModelError("", $"Unable to list pools. Error: {ex}");
}
return(View(model));
}
public async Task <IReadOnlyList <RenderingEnvironment> > Environments()
{
var envs = await Task.WhenAll((await _environmentCoordinator.ListEnvironments())
.Select(env => _environmentCoordinator.GetEnvironment(env)));
return(envs.Where(re => re != null).OrderBy(re => re.Name).ToList());
}
public async Task OnAuthorizationAsync(AuthorizationFilterContext context)
{
if (context.HttpContext.Request.Headers.ContainsKey("Authorization"))
{
var authHeader = context.HttpContext.Request.Headers["Authorization"];
if (context.RouteData.Values.ContainsKey("environmentName") &&
context.RouteData.Values["environmentName"] is string envName)
{
if (!string.IsNullOrEmpty(envName))
{
var env = await _environmentCoordinator.GetEnvironment(envName);
if (env != null &&
env.Enabled &&
env.AutoScaleConfiguration != null &&
env.AutoScaleConfiguration.ScaleEndpointEnabled)
{
if (!string.IsNullOrWhiteSpace(env.AutoScaleConfiguration.PrimaryApiKey) &&
!string.IsNullOrWhiteSpace(env.AutoScaleConfiguration.SecondaryApiKey))
{
if (authHeader == $"Basic {env.AutoScaleConfiguration.PrimaryApiKey}" ||
authHeader == $"Basic {env.AutoScaleConfiguration.SecondaryApiKey}")
{
return;
}
}
}
}
}
}
context.Result = new StatusCodeResult((int)HttpStatusCode.Forbidden);
}
public async Task <ActionResult> ScaleUpPool(string environmentName, string poolName, [FromBody] Models.Api.ScaleUpRequest request)
{
var environment = await _environmentCoordinator.GetEnvironment(environmentName);
if (environment == null)
{
return(NotFound());
}
// TODO: should we validate the pool here as well?
try
{
await _scaleUpStore.Add(
environment.Name,
poolName,
request.RequestedNodes);
// tell scale up processor to start now, bypass delay
_trigger.Set();
return(Ok());
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed to add scale entry to table storage for environment: {environmentName} and pool: {poolName}");
throw;
}
}
public async Task <ActionResult> EnvironmentCosts(string environmentName, [FromQuery] DateTimeOffset?from, [FromQuery] DateTimeOffset?to)
{
var environment = await _environmentCoordinator.GetEnvironment(environmentName);
if (environment == null)
{
return(NotFound());
}
var period = new CostPeriod(nameof(EnvironmentCosts), Url, from, to);
var cost = await _costCoordinator.GetCost(environment, period.QueryTimePeriod);
cost.CurrentMonthLink = period.GetCurrentMonthLink();
cost.NextMonthLink = period.GetNextMonthLink();
cost.PreviousMonthLink = period.GetPrevMonthLink();
return(Ok(cost));
}
public async Task <ActionResult> EnvironmentPoolsUsage(string environmentName)
{
var environment = await _environmentCoordinator.GetEnvironment(environmentName);
if (environment == null)
{
return(NotFound());
}
var usage = await _poolUsageProvider.GetEnvironmentUsage(environment);
return(Ok(usage));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var interval = TimeSpan.FromSeconds(60);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var environments = await _environments.ListEnvironments();
await Task.WhenAll(environments.Select(async e => await AutoScaleEnvironment(await _environments.GetEnvironment(e))));
}
catch (Exception e)
{
Console.WriteLine(e);
}
await Task.Delay(interval, stoppingToken);
}
}
private async Task <RequestStatus> PerformRequest(ScaleUpRequestEntity request)
{
var env = await _envs.GetEnvironment(request.EnvironmentName);
if (env == null)
{
_logger.LogInformation(
"Environment '{0}' has been deleted, discarding scale request '{1}'",
request.EnvironmentName,
request.ETag);
return(RequestStatus.Completed);
}
using (var batchClient = await _clientProvider.CreateBatchManagementClient(env.SubscriptionId))
{
try
{
var pool = await batchClient.Pool.GetAsync(env.BatchAccount.ResourceGroupName, env.BatchAccount.Name, request.PoolName);
if (pool == null || pool.ProvisioningState == PoolProvisioningState.Deleting)
{
_logger.LogInformation(
"Pool '{0}' (in environment '{1}') has been deleted, discarding scale request '{2}'",
request.PoolName,
request.EnvironmentName,
request.ETag);
return(RequestStatus.Completed);
}
if (pool.AllocationState == AllocationState.Resizing)
{
var op = pool.ResizeOperationStatus;
if (op != null && ((op.TargetDedicatedNodes ?? 0) + (op.TargetLowPriorityNodes ?? 0)) >= request.TargetNodes)
{
_logger.LogInformation(
"A resize operation on pool '{0}' (in environment '{1}') has made scale request '{2}' redundant, discarding it",
request.PoolName,
request.EnvironmentName,
request.ETag);
return(RequestStatus.Completed);
}
else
{
_logger.LogInformation(
"Pool '{0}' (in environment '{1}') is already being resized. Waiting to apply scale request '{2}'",
request.PoolName,
request.EnvironmentName,
request.ETag);
return(RequestStatus.InProgress);
}
}
var targets = CalculateNodeTargets(request, pool);
var newPool =
new Pool(name: pool.Name)
{
ScaleSettings =
new ScaleSettings
{
FixedScale =
new FixedScaleSettings(
targetLowPriorityNodes: targets.lowPriority,
targetDedicatedNodes: targets.dedicated)
}
};
await batchClient.Pool.UpdateAsync(env.BatchAccount.ResourceGroupName, env.BatchAccount.Name, request.PoolName, newPool);
_logger.LogInformation(
"Successfully applied scale request '{0}' to pool '{1}' (in environment '{2}')",
request.ETag,
request.PoolName,
request.EnvironmentName);
}
catch (CloudException ce) when(ce.ResourceNotFound())
{
// Pool is gone - complete the request to remove it.
}
return(RequestStatus.Completed);
}
}
public Task <RenderingEnvironment> Environment(string envId) => _environmentCoordinator.GetEnvironment(envId);
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var interval = TimeSpan.FromSeconds(60);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var environments = await _environments.ListEnvironments();
await Task.WhenAll(environments.Select(async e => await AutoScaleEnvironment(await _environments.GetEnvironment(e))));
}
catch (Exception ex)
{
_logger.LogError(ex, $"Unexpected error in {nameof(AutoScaleHost)} main loop");
}
await Task.Delay(interval, stoppingToken);
}
}
