public async Async.Task Halt(Scaleset scaleset)
{
var shrinkQueue = new ShrinkQueue(scaleset.ScalesetId, _context.Queue, _log);
await shrinkQueue.Delete();
await foreach (var node in _context.NodeOperations.SearchStates(scaleSetId: scaleset.ScalesetId))
{
_log.Info($"{SCALESET_LOG_PREFIX} deleting node scaleset_id {scaleset.ScalesetId} machine_id {node.MachineId}");
await _context.NodeOperations.Delete(node);
}
_log.Info($"{SCALESET_LOG_PREFIX} scaleset delete starting: scaleset_id:{scaleset.ScalesetId}");
if (await _context.VmssOperations.DeleteVmss(scaleset.ScalesetId))
{
_log.Info($"{SCALESET_LOG_PREFIX}scaleset deleted: scaleset_id {scaleset.ScalesetId}");
var r = await Delete(scaleset);
if (!r.IsOk)
{
_log.WithHttpStatus(r.ErrorV).Error($"Failed to delete scaleset record {scaleset.ScalesetId}");
}
}
else
{
var r = await Replace(scaleset);
if (!r.IsOk)
{
_log.WithHttpStatus(r.ErrorV).Error($"Failed to save scaleset record {scaleset.ScalesetId}");
}
}
}
public async Async.Task <Pool> Init(Pool pool)
{
await _context.Queue.CreateQueue(GetPoolQueue(pool.PoolId), StorageType.Corpus);
var shrinkQueue = new ShrinkQueue(pool.PoolId, _context.Queue, _logTracer);
await shrinkQueue.Create();
await SetState(pool, PoolState.Running);
return(pool);
}
public async Async.Task SetSize(Scaleset scaleset, int size)
{
// # no longer needing to resize
if (scaleset is null)
{
return;
}
if (scaleset.State != ScalesetState.Resize)
{
return;
}
_log.Info($"{SCALESET_LOG_PREFIX} scaleset resize: scaleset_id:{scaleset.ScalesetId} size:{size}");
var shrinkQueue = new ShrinkQueue(scaleset.ScalesetId, _context.Queue, _log);
// # reset the node delete queue
await shrinkQueue.Clear();
//# just in case, always ensure size is within max capacity
scaleset = scaleset with {
Size = Math.Min(scaleset.Size, MaxSize(scaleset))
};
// # Treat Azure knowledge of the size of the scaleset as "ground truth"
var vmssSize = await _context.VmssOperations.GetVmssSize(scaleset.ScalesetId);
if (vmssSize is null)
{
_log.Info($"{SCALESET_LOG_PREFIX} scaleset is unavailable. scaleset_id {scaleset.ScalesetId}");
//#if the scaleset is missing, this is an indication the scaleset
//# was manually deleted, rather than having OneFuzz delete it. As
//# such, we should go thruogh the process of deleting it.
await SetShutdown(scaleset, now : true);
return;
}
else if (scaleset.Size == vmssSize)
{
await ResizeEqual(scaleset);
}
else if (scaleset.Size > vmssSize)
{
await ResizeGrow(scaleset);
}
else
{
await ResizeShrink(scaleset, vmssSize - scaleset.Size);
}
}
public async Async.Task <Pool> Halt(Pool pool)
{
//halt the pool immediately
var scalesets = _context.ScalesetOperations.SearchByPool(pool.Name);
var nodes = _context.NodeOperations.SearchByPoolName(pool.Name);
if (scalesets is null && nodes is null)
{
var poolQueue = GetPoolQueue(pool.PoolId);
await _context.Queue.DeleteQueue(poolQueue, StorageType.Corpus);
var shrinkQueue = new ShrinkQueue(pool.PoolId, _context.Queue, _logTracer);
await shrinkQueue.Delete();
_logTracer.Info($"pool stopped, deleting: {pool.Name}");
var r = await Delete(pool);
if (!r.IsOk)
{
_logTracer.Error($"Failed to delete pool: {pool.Name} due to {r.ErrorV}");
}
}
if (scalesets is not null)
{
await foreach (var scaleset in scalesets)
{
if (scaleset is not null)
{
await _context.ScalesetOperations.SetState(scaleset, ScalesetState.Halt);
}
}
}
if (nodes is not null)
{
await foreach (var node in nodes)
{
await _context.NodeOperations.SetHalt(node);
}
}
return(pool);
}
public async Task <bool> CouldShrinkScaleset(Node node)
{
if (node.ScalesetId is Guid scalesetId)
{
var queue = new ShrinkQueue(scalesetId, _context.Queue, _logTracer);
if (await queue.ShouldShrink())
{
return(true);
}
}
if (node.PoolId is Guid poolId)
{
var queue = new ShrinkQueue(poolId, _context.Queue, _logTracer);
if (await queue.ShouldShrink())
{
return(true);
}
}
return(false);
}
