private static void DisplayInstanceResults(InstanceResults results)
{
var hasBadInstances = results.BadInstances != null && results.BadInstances.Count > 0;
if (ShowGoodInstances || ShowUnknownInstances || hasBadInstances)
{
Console.WriteLine("");
}
if (ShowGoodInstances)
{
Console.ForegroundColor = ConsoleColor.Green;
WriteStat($"Good Instances ({results.GoodInstances.Count})", "CPU%");
Console.WriteLine("=====================================================================================");
foreach (var result in results.GoodInstances)
{
WriteStat(result.Name, result.Cpu);
}
Console.WriteLine("");
}
if (ShowUnknownInstances)
{
Console.ForegroundColor = ConsoleColor.Cyan;
WriteStat($"Unknown Instances ({results.UnknownInstances.Count})", "CPU%");
Console.WriteLine("=================================================================================");
foreach (var result in results.UnknownInstances)
{
WriteStat(result.Name, result.Cpu);
}
Console.WriteLine("");
}
if (hasBadInstances)
{
Console.ForegroundColor = ConsoleColor.Red;
WriteStat($"Bad Instances ({results.BadInstances.Count})", "Reason", "CPU%");
Console.WriteLine("===================================================================================================");
foreach (var result in results.BadInstances)
{
WriteStat(result.Name, result.Reason, result.Cpu);
}
Console.WriteLine("");
}
Console.ForegroundColor = ConsoleColor.White;
}
private static void DisplayInstanceResults(InstanceResults results)
{
Console.WriteLine("");
if (ShowGoodInstances)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Good Instances ({0})\t\t\t\t\t\t\t\tCPU%", results.GoodInstances.Count);
Console.WriteLine("=====================================================================================");
foreach (var result in results.GoodInstances)
{
Console.WriteLine(result);
}
Console.WriteLine("");
}
if (ShowUnknownInstances)
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Unknown Instances ({0})\t\t\t\t\t\t\t\tCPU%", results.UnknownInstances.Count);
Console.WriteLine("=================================================================================");
foreach (var result in results.UnknownInstances)
{
Console.WriteLine(result);
}
Console.WriteLine("");
}
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Bad Instances ({0})\t\t\t\t\t\t\t\tCPU%", results.BadInstances.Count);
Console.WriteLine("=====================================================================================");
foreach (var result in results.BadInstances)
{
Console.WriteLine(result);
}
Console.WriteLine("");
Console.ForegroundColor = ConsoleColor.White;
}
private static async Task <InstanceResults> FindInstancesAndCategorize()
{
var instances = await FindInstances();
AmazonCloudWatchClient cwClient = new AmazonCloudWatchClient(AccessKey, SecretKey, Region);
var instanceResults = new InstanceResults();
//this was written this way to facilite better control over throttling (vs Parallel.ForEach with MaxConcurrency)
//in addition a future enhancement could be batching instances to GetMatricStatistics call
await instances.ForEachAsync(MaxMetricConcurrency, async (instance) =>
{
//var instance = instances[i];
var dim = new Dimension()
{
Name = "InstanceId",
Value = instance.InstanceId,
};
var request = new GetMetricStatisticsRequest()
{
Dimensions = new List <Dimension> {
dim
},
Namespace = "AWS/EC2",
MetricName = "CPUUtilization",
Unit = StandardUnit.Percent,
Statistics = new List <string>()
{
"Average"
},
StartTime = DateTime.UtcNow.Subtract(TimeSpan.FromHours(MetricRetrievalInHours)),
EndTime = DateTime.UtcNow,
Period = (int)TimeSpan.FromMinutes(PeriodInMinutes).TotalSeconds,
};
var response = await cwClient.GetMetricStatisticsAsync(request);
var max = UnknownCpuPercent;
var avg = 0.0d;
var stdDev = 0.0d;
var count = response.Datapoints.Count;
if (count >= MinServerMetricsToJudge)
{
// first loop to calculate max and sum
var sum = 0.0d;
foreach (var dp in response.Datapoints)
{
if (dp.Average > max)
{
max = dp.Average;
}
sum += dp.Average;
}
avg = sum / count;
// second loop allows us to calculate std deviation
var varianceSum = 0.0d;
foreach (var dp in response.Datapoints)
{
varianceSum += Math.Pow(dp.Average - avg, 2.0d);
}
stdDev = Math.Sqrt(varianceSum / count);
}
if (max == UnknownCpuPercent)
{
instanceResults.UnknownInstances.Add(new InstanceResult(instance, "Unknown CPU", max));
}
else if (max < MaxBadCpuPercent)
{
// if the CPU is basically doing nothing
//Console.WriteLine($"LOW {instance.Hostname}\tMax={max}, Avg={avg}, StdDev={stdDev}");
instanceResults.BadInstances.Add(new InstanceResult(instance, "Low CPU", max));
}
else if (max > MinStalledCpuPercent && stdDev < MaxStdDevStalledCpuPercent)
{
// if the CPU is above a floor (MinStalledCpuPercent) but is essentially flatlined
//Console.WriteLine($"FLAT {instance.Hostname}\tMax={max}, Avg={avg}, StdDev={stdDev}");
instanceResults.BadInstances.Add(new InstanceResult(instance, "Flatlined CPU", stdDev));
}
else
{
instanceResults.GoodInstances.Add(new InstanceResult(instance, null, max));
}
var processed = instanceResults.BadInstances.Count + instanceResults.UnknownInstances.Count +
instanceResults.GoodInstances.Count;
Console.Write("\rProcessed {0}/{1} instances.", processed, instances.Count);
//depending on your AWS setup you might need to throttle these calls more aggressively
//Thread.Sleep(50);
});
Console.WriteLine();
return(instanceResults);
}
private static async Task <InstanceResults> FindInstancesAndCategorize()
{
var instances = await FindInstances();
AmazonCloudWatchClient cwClient = new AmazonCloudWatchClient(AccessKey, SecretKey, Region);
var instanceResults = new InstanceResults();
//this was written this way to facilite better control over throttling (vs Parallel.ForEach with MaxConcurrency)
//in addition a future enhancement could be batching instances to GetMatricStatistics call
await instances.ForEachAsync(MaxMetricConcurrency, async (instance) =>
{
//var instance = instances[i];
var dim = new Dimension()
{
Name = "InstanceId",
Value = instance.InstanceId,
};
var request = new GetMetricStatisticsRequest()
{
Dimensions = new List <Dimension> {
dim
},
Namespace = "AWS/EC2",
MetricName = "CPUUtilization",
Unit = StandardUnit.Percent,
Statistics = new List <string>()
{
"Average"
},
StartTime = DateTime.UtcNow.Subtract(TimeSpan.FromHours(MetricRetrievalInHours)),
EndTime = DateTime.UtcNow,
Period = (int)TimeSpan.FromMinutes(PeriodInMinutes).TotalSeconds,
};
var response = await cwClient.GetMetricStatisticsAsync(request);
var max = response.Datapoints.Count < MinServerMetricsToJudge ? UnknownCpuPercent : response.Datapoints.Max(p => p.Average);
if (max == UnknownCpuPercent)
{
instanceResults.UnknownInstances.Add(new InstanceResult(instance, max));
}
else if (max < MaxBadCpuPercent)
{
instanceResults.BadInstances.Add(new InstanceResult(instance, max));
}
else
{
instanceResults.GoodInstances.Add(new InstanceResult(instance, max));
}
var processed = instanceResults.BadInstances.Count + instanceResults.UnknownInstances.Count +
instanceResults.GoodInstances.Count;
Console.Write("\rProcessed {0}/{1} instances.", processed, instances.Count);
//depending on your AWS setup you might need to throttle these calls more aggressively
//Thread.Sleep(50);
});
Console.WriteLine();
return(instanceResults);
}
