internal static void ShutDown(ILogger log, List <WorkloadMachineTuple> totalWorkload, BBBAPI bbb, IonosAPI ionos, string ionosDataCenter)
{
var shutDown = totalWorkload
.OrderByDescending(m => m.Workload.MemoryUtilization)
.ThenByDescending(m => m.Workload.CPUUtilization)
.Skip(MINIMUM_ACTIVE_MACHINES)
.Select(async m => new MachineWorkloadStatsTuple(m.Machine, m.Workload, await bbb.GetMeetingsAsync(m.Machine.PrimaryIP).ConfigureAwait(false)))
.Select(res => res.Result)
.Where(m => (m.Workload.MemoryUtilization < MIN_ALLOWED_MEMORY_WORKLOAD ||
m.Workload.CPUUtilization < MIN_ALLOWED_CPU_WORKLOAD) && m.Machine.Properties.Cores <= DEFAULT_WORKER_CPU &&
m.Machine.Properties.Ram <= DEFAULT_WORKER_MEMORY && m.Stats.Sum(u => u.ParticipantCount) == 0).ToList();
log.LogInformation($"Found '{shutDown.Count}' machines to shut down");
shutDown.AsParallel().ForAll(async m => await ionos.TurnMachineOff(m.Machine.Id, ionosDataCenter).ConfigureAwait(false));
}
async Task StartJob(CancellationToken cancellationToken)
{
log.LogInformation("Start autoscaler job");
string ionosUser = Environment.GetEnvironmentVariable("IONOS_USER");
string ionosPass = Environment.GetEnvironmentVariable("IONOS_PASS");
string ionosDataCenter = Environment.GetEnvironmentVariable("IONOS_DATACENTER");
string bbbKey = Environment.GetEnvironmentVariable("BBB_PASS");
string graphanaKey = Environment.GetEnvironmentVariable("GRAFANA_PASS");
string neUser = Environment.GetEnvironmentVariable("NE_BASIC_AUTH_USER");
string nePass = Environment.GetEnvironmentVariable("NE_BASIC_AUTH_PASS");
if (string.IsNullOrEmpty(ionosUser) || string.IsNullOrEmpty(ionosPass) || string.IsNullOrEmpty(ionosDataCenter))
{
log.LogInformation("Load dev environment variables");
ionosUser = ConfigReader.GetConfigurationValue("IONOS", "USER");
ionosPass = ConfigReader.GetConfigurationValue("IONOS", "PASS");
ionosDataCenter = ConfigReader.GetConfigurationValue("IONOS", "DATACENTER");
bbbKey = ConfigReader.GetConfigurationValue("BBB", "PASS");
graphanaKey = ConfigReader.GetConfigurationValue("GRAFANA", "PASS");
neUser = ConfigReader.GetConfigurationValue("NODE_EXPORTER", "USER");
nePass = ConfigReader.GetConfigurationValue("NODE_EXPORTER", "PASS");
}
log.LogInformation("Init Ionos API");
IonosAPI ionos = new IonosAPI(log, ionosUser, ionosPass);
while (!cancellationToken.IsCancellationRequested)
{
try
{
log.LogInformation("Get internet ip");
using var web = new WebClient();
string externalip = web.DownloadString("http://icanhazip.com")
.Replace("\n", "", true, CultureInfo.InvariantCulture)
.Replace("\r", "", true, CultureInfo.InvariantCulture)
.Trim();
#if DEBUG
externalip = "81.173.115.126";
#endif
//Get all machines
log.LogInformation("Get machines");
var ml = await ionos.GetAllMachines(ionosDataCenter).ConfigureAwait(false);
//Get IPs
log.LogInformation("Get machine details");
var machines = ml.AsParallel().Select(async m => await ionos.GetMachineDetails(m.Id, ionosDataCenter).ConfigureAwait(false)).Select(m => m.Result)
.Select(async m =>
{
if (m.Entities.Nics.Items.Count > 0)
{
m.PrimaryIP = m.Entities.Nics.Items
.FirstOrDefault(p => p.Properties.Name.ToUpperInvariant() == "public".ToUpperInvariant())
.Properties.Ips.FirstOrDefault();
}
if (m.Entities.Nics.Items.Count > 1)
{
m.SecondaryIP = m.Entities.Nics.Items
.FirstOrDefault(p => p.Properties.Name.ToUpperInvariant() != "public".ToUpperInvariant())
.Properties.Ips.FirstOrDefault();
}
return(m);
})
.Select(m => m.Result)
//Filter for Scalelite
.Where(m => !m.Properties.Name.ToUpperInvariant().Contains("SCALELITE", StringComparison.InvariantCultureIgnoreCase) &&
m.PrimaryIP != null)
.ToList();
log.LogInformation("Get running machines");
var runningMachines = machines.Where(m => m.Properties.VmState == "RUNNING").ToList();
log.LogInformation("Get workload of machines");
var totalWorkload = runningMachines.AsParallel()
.Select(async m => new WorkloadMachineTuple(m, await NodeExporterAPI.GetWorkLoadAsync(log, m.SecondaryIP ?? m.PrimaryIP, graphanaKey, neUser, nePass).ConfigureAwait(false)))
.Select(m => m.Result).ToList();
float avarageMemoryWorkload = totalWorkload.Average(m => m.Workload.MemoryUtilization);
float avarageCPUWorkload = totalWorkload.Average(m => m.Workload.CPUUtilization);
log.LogInformation($"Average workload is '{avarageMemoryWorkload}'");
log.LogInformation($"Max allowed workload is '{MAX_ALLOWED_MEMORY_WORKLOAD}'");
if (avarageMemoryWorkload >= MAX_ALLOWED_MEMORY_WORKLOAD || avarageCPUWorkload >= MAX_ALLOWED_CPU_WORKLOAD)
{
log.LogInformation("Wake up sleeping machine");
var sleepingMachines = machines.Where(m => !runningMachines.Contains(m));
var toBeTurnedOn = sleepingMachines.FirstOrDefault();
if (toBeTurnedOn != null)
{
await ionos.TurnMachineOn(toBeTurnedOn.Id, ionosDataCenter).ConfigureAwait(false);
}
}
//Shut all machines down that have no more memory to reduce an broken the threshold
log.LogInformation("Connect to host and get current sessions");
BBBAPI bbb = new BBBAPI(log, bbbKey);
ScalingHelper.ShutDown(log, totalWorkload, bbb, ionos, ionosDataCenter);
//Scale all machines down that have addional memory received
ScalingHelper.ScaleMemoryDown(log, totalWorkload, ionos, ionosDataCenter);
//Scale all machines down that have addional cpu received
ScalingHelper.ScaleCPUDown(log, totalWorkload, ionos, ionosDataCenter);
//Scale all machines up thats memory are over the max workload threshold
ScalingHelper.ScaleMemoryUp(log, totalWorkload, ionos, ionosDataCenter);
//Scale all machines up thats cpu utilization is over the max workload threshold
ScalingHelper.ScaleCPUUp(log, totalWorkload, ionos, ionosDataCenter);
log.LogInformation($"Sleep for '{WAITINGTIME}' milliseconds ('{WAITINGTIME / 60000} minutes')");
Thread.Sleep(WAITINGTIME);
}
catch (Exception e)
{
log.LogError(e, "Exepthion thrown:");
log.LogInformation($"Sleep for '{WAITINGTIME}' milliseconds ('{WAITINGTIME / 60000} minutes')");
Thread.Sleep(WAITINGTIME);
}
}
}
internal static void ScaleCPUDown(ILogger log, List <WorkloadMachineTuple> totalWorkload, BBBAPI bbb, IonosAPI ionos)
{
var scaleCpuDown = totalWorkload.OrderByDescending(m => m.Workload.CPUUtilization)
.Select(async m => new MachineWorkloadStatsTuple(m.DataCenter, m.Machine, m.Workload, await bbb.GetMeetingsAsync(m.Machine.PrimaryIP).ConfigureAwait(false)))
.Select(res => res.Result)
.Where(m => m.Workload.CPUUtilization <MIN_ALLOWED_CPU_WORKLOAD &&
m.Machine.Properties.Cores> DEFAULT_WORKER_CPU && m.Stats.Sum(u => u.ParticipantCount) == 0).ToList();
log.LogInformation($"Found '{scaleCpuDown.Count}' machines to scale CPU down");
scaleCpuDown.AsParallel().ForAll(async m =>
{
var machine = m.Machine;
log.LogInformation($"Scale machine '{machine.PrimaryIP}' down");
var update = new IonosMachineUpdate {
Cores = machine.Properties.Cores - 1
};
await ionos.UpdateMachines(machine.Id, m.DataCenter, update).ConfigureAwait(false);
});
}
async Task StartJob(CancellationToken cancellationToken)
{
log.LogInformation("Start autoscaler job");
string ionosUser = Environment.GetEnvironmentVariable("IONOS_USER");
string ionosPass = Environment.GetEnvironmentVariable("IONOS_PASS");
string[] ionosDataCenterIds = Environment.GetEnvironmentVariable("IONOS_DATACENTER").Split(",");
string bbbKey = Environment.GetEnvironmentVariable("BBB_PASS");
string graphanaKey = Environment.GetEnvironmentVariable("GRAFANA_PASS");
string neUser = Environment.GetEnvironmentVariable("NE_BASIC_AUTH_USER");
string nePass = Environment.GetEnvironmentVariable("NE_BASIC_AUTH_PASS");
if (string.IsNullOrEmpty(ionosUser) || string.IsNullOrEmpty(ionosPass) || ionosDataCenterIds == null || ionosDataCenterIds.Any(id => string.IsNullOrEmpty(id)))
{
log.LogInformation("Load dev environment variables");
ionosUser = ConfigReader.GetConfigurationValue("IONOS", "USER");
ionosPass = ConfigReader.GetConfigurationValue("IONOS", "PASS");
ionosDataCenterIds = ConfigReader.GetConfigurationValue("IONOS", "DATACENTER").Split(",");
bbbKey = ConfigReader.GetConfigurationValue("BBB", "PASS");
graphanaKey = ConfigReader.GetConfigurationValue("GRAFANA", "PASS");
neUser = ConfigReader.GetConfigurationValue("NODE_EXPORTER", "USER");
nePass = ConfigReader.GetConfigurationValue("NODE_EXPORTER", "PASS");
}
log.LogInformation("Init Ionos API");
IonosAPI ionos = new IonosAPI(log, ionosUser, ionosPass);
while (!cancellationToken.IsCancellationRequested)
{
try
{
ScalingHelper.EnsureMinimumRunningServers(log, ionos, ionosDataCenterIds);
log.LogInformation("Get machines");
var machines = await IonosHelper.GetMachinesByDataCenter(log, ionos, ionosDataCenterIds);
var runningMachines = machines.Where(m => m.Machine.Properties.VmState == "RUNNING");
var totalWorkload = runningMachines.AsParallel()
.Select(async m => new WorkloadMachineTuple(m.DataCenter, m.Machine, await NodeExporterAPI.GetWorkLoadAsync(log, m.Machine.SecondaryIP ?? m.Machine.PrimaryIP, graphanaKey, neUser, nePass).ConfigureAwait(false)))
.Select(m => m.Result).ToList();
float avarageMemoryWorkload = totalWorkload.Average(m => m.Workload.MemoryUtilization);
float avarageCPUWorkload = totalWorkload.Average(m => m.Workload.CPUUtilization);
log.LogInformation($"Average workload is '{avarageMemoryWorkload}'");
log.LogInformation($"Max allowed workload is '{MAX_ALLOWED_MEMORY_WORKLOAD}'");
if (avarageMemoryWorkload >= MAX_ALLOWED_MEMORY_WORKLOAD || avarageCPUWorkload >= MAX_ALLOWED_CPU_WORKLOAD)
{
log.LogInformation("Wake up sleeping machine");
var sleepingMachines = machines.Where(m => m.Machine.Properties.VmState == "SHUTOFF");
var toBeTurnedOn = sleepingMachines.FirstOrDefault();
if (toBeTurnedOn != null)
{
await ionos.TurnMachineOn(toBeTurnedOn.Machine.Id, toBeTurnedOn.DataCenter).ConfigureAwait(false);
}
}
BBBAPI bbb = new BBBAPI(log, bbbKey);
ScalingHelper.ShutDown(log, totalWorkload, bbb, ionos);
//Scale all machines down that have addional memory received
ScalingHelper.ScaleMemoryDown(log, totalWorkload, bbb, ionos);
//Scale all machines down that have addional cpu received
ScalingHelper.ScaleCPUDown(log, totalWorkload, bbb, ionos);
//Scale all machines up thats memory are over the max workload threshold
ScalingHelper.ScaleMemoryUp(log, totalWorkload, ionos);
//Scale all machines up thats cpu utilization is over the max workload threshold
ScalingHelper.ScaleCPUUp(log, totalWorkload, ionos);
log.LogInformation($"Sleep for '{WAITINGTIME}' milliseconds ('{WAITINGTIME / 60000} minutes')");
Thread.Sleep(WAITINGTIME);
}
catch (Exception e)
{
log.LogError(e, "Exeption thrown:");
log.LogInformation($"Sleep for '{WAITINGTIME}' milliseconds ('{WAITINGTIME / 60000} minutes')");
Thread.Sleep(WAITINGTIME);
}
}
}
