static void Main(string[] args)
{
var droneApi = new DroneApi(Env.DroneUrl, Env.DroneToken);
var lastDroneApiSuccess = DateTime.MinValue;
var allAgentsAreTerminated = false;
while (true)
{
var droneQueue = new DroneQueue();
try {
droneQueue = droneApi.ReadQueue();
lastDroneApiSuccess = DateTime.Now;
} catch (Exception x) {
PrintError(x);
if (DateTime.Now - lastDroneApiSuccess < TimeSpan.FromMinutes(5))
{
Console.WriteLine(" Will retry");
PauseWithDots();
continue;
}
else
{
Console.WriteLine(" Assuming queue is empty");
}
}
// FYI https://github.com/drone/drone/issues/2189
foreach (var b in droneQueue.ZombieBuilds)
{
Kill(b, "zombie");
}
// https://github.com/drone/drone/issues/2484
foreach (var b in droneQueue.CancelledBuilds)
{
Kill(b, "cancelled");
}
void Kill(DroneBuild b, string reason)
{
try {
droneApi.KillBuild(b, reason);
} catch (Exception x) {
PrintError(x);
}
}
var desiredAgentCount = CalcDesiredAgentCount(droneQueue.ActiveJobCount);
Console.WriteLine("Active jobs: " + droneQueue.ActiveJobCount);
Console.WriteLine("Desired agents: " + desiredAgentCount);
if (desiredAgentCount > 0 || !allAgentsAreTerminated)
{
AdjustAgentCount(desiredAgentCount, out allAgentsAreTerminated);
}
else
{
Console.WriteLine("Do nothing");
}
PauseWithDots();
}
}
public DroneQueue ReadQueue()
{
// FYI https://github.com/drone/drone/issues/2251
var result = new DroneQueue();
using (var web = new WebClient()) {
Console.WriteLine("Reading Drone jobs");
foreach (var b in ReadBuilds(web))
{
Console.Write($" {b.owner}/{b.name} #{b.number} ");
var createdAt = DateTimeOffset.FromUnixTimeSeconds((long)b.created_at);
if ((DateTime.UtcNow - createdAt).TotalHours > 6)
{
Console.WriteLine($"too old, created at {createdAt:u}");
// Don't count too old builds
// They may be abnormal queue items that are never picked up by an agent
continue;
}
var zombie = true;
var procs = ReadBuildDetails(web, b).procs as IEnumerable <dynamic>;
if (procs != null)
{
var hasCancelledProc = procs.Any(p => p.state == "killed" || p.state == "failure" && p.error == "Cancelled");
if (hasCancelledProc)
{
Console.WriteLine("has killed or cancelled proc");
result.CancelledBuilds.Add(DroneBuild.FromApiObject(b));
continue;
}
foreach (var p in procs)
{
Console.Write(((string)p.state)[0]);
if (p.state == "running" || p.state == "pending")
{
result.ActiveJobCount++;
zombie = false;
}
}
}
else
{
Console.Write("no procs!");
zombie = false; // TODO
}
if (zombie)
{
Console.Write(" [zombie]");
result.ZombieBuilds.Add(DroneBuild.FromApiObject(b));
}
Console.WriteLine();
}
}
return(result);
}
