public static void Validate(WatchmanConfiguration config)
{
if (config == null)
{
throw new ConfigException("Config cannot be null");
}
if (!HasAny(config.AlertingGroups))
{
throw new ConfigException("Config must have alerting groups");
}
foreach (var alertingGroup in config.AlertingGroups)
{
Validate(alertingGroup);
}
var duplicateNames = config.AlertingGroups
.Select(g => g.Name)
.GroupBy(_ => _)
.Where(g => g.Count() > 1)
.Select(g => g.Key)
.ToList();
if (duplicateNames.Any())
{
throw new ConfigException($"The following alerting group names exist in multiple config files: {string.Join(", ", duplicateNames)}");
}
}
public async Task <GenerateAlarmsResult> GenerateAlarmsForService(
WatchmanConfiguration config, RunMode mode)
{
var serviceConfig = _serviceConfigMapper(config);
// hack to make sure the alarm creator knows about all groups
foreach (var group in serviceConfig.AlertingGroups)
{
_creator.AddAlarms(group.GroupParameters, new List <Alarm>());
}
if (!ServiceConfigIsPopulated(serviceConfig))
{
_logger.Info($"No resources for {serviceConfig.ServiceName}. No action taken for this resource type");
return(new GenerateAlarmsResult());
}
var populatedServiceConfig = await PopulateResourceNames(serviceConfig);
var failures = await GenerateAlarms(populatedServiceConfig, mode);
await ReportOrphans(populatedServiceConfig);
return(new GenerateAlarmsResult(failures));
}
public void Setup()
{
_config = ConfigTestData.ValidConfig();
_config.AlertingGroups.First().DynamoDb.Tables = new List <Table>
{
new Table {
Name = "TableName"
}
};
_config.AlertingGroups.First().Sqs = new Sqs
{
Queues = new List <Queue>
{
new Queue {
Name = "QueueName"
}
}
};
_config.AlertingGroups.First().Services = new Dictionary <string, AwsServiceAlarms>
{
{
"ServiceName", new AwsServiceAlarms
{
Resources = new List <ResourceThresholds>
{
new ResourceThresholds {
Pattern = "ResourceName"
}
}
}
}
};
}
public async Task DoesNotThrowIfTableIsReturnedInListingButCannotBeDescribed()
{
var mockery = new DynamoAlarmGeneratorMockery();
var generator = mockery.AlarmGenerator;
mockery.GivenAListOfTables(new[] { "banana", "apple" });
mockery.GivenATable("apple", 1300, 600);
mockery.ValidSnsTopic();
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "TestGroup",
AlarmNameSuffix = "TestGroup",
DynamoDb = new DynamoDb()
{
Tables = new List <Table>
{
new Table {
Pattern = "^.*$"
}
}
}
}
}
};
await generator.GenerateAlarmsFor(config, RunMode.GenerateAlarms);
}
public void Setup()
{
_config = ConfigTestData.ValidConfig();
_config.AlertingGroups.First().DynamoDb.Tables = new List <Table>
{
new Table {
Name = "TableName"
}
};
_config.AlertingGroups.First().Sqs = new Sqs
{
Queues = new List <Queue>
{
new Queue {
Name = "QueueName"
}
}
};
_config.AlertingGroups.First().Services = new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >
{
new ResourceThresholds <AutoScalingResourceConfig> {
Pattern = "ResourceName"
}
}
}
};
}
private async Task GenerateAlarms(WatchmanConfiguration config, RunMode mode)
{
await _dynamoGenerator.GenerateAlarmsFor(config, mode);
await _orphanTablesReporter.FindAndReport(config);
await _sqsGenerator.GenerateAlarmsFor(config, mode);
await _orphanQueuesReporter.FindAndReport(config);
var failed = new List <IList <string> >();
foreach (var service in _otherServices)
{
var result = await service.GenerateAlarmsForService(config, mode);
failed.Add(result.FailingGroups);
}
var allFailed = failed.SelectMany(_ => _).Distinct().ToArray();
if (mode == RunMode.DryRun || mode == RunMode.GenerateAlarms)
{
await _creator.SaveChanges(mode == RunMode.DryRun);
}
if (allFailed.Any())
{
throw new Exception("The following groups reported errors and were not deployed: " +
$"{string.Join(", ", allFailed)}. please see logs.");
}
}
public void Setup()
{
_awsServiceAlarms = new AwsServiceAlarms <ResourceConfig>
{
ExcludeResourcesPrefixedWith = new List <string>
{
"ExcludePrefix"
},
Resources = new List <ResourceThresholds <ResourceConfig> >
{
new ResourceThresholds <ResourceConfig>
{
Name = "ResourceName",
Values = new Dictionary <string, AlarmValues>
{
{ "testThresholdLow", 42 }
}
}
},
Values = new Dictionary <string, AlarmValues>
{
{ "testThresholdHigh", 242 }
}
};
_config = ConfigTestData.ValidConfig();
_config.AlertingGroups.First().Services = new AlertingGroupServices()
{
Lambda = _awsServiceAlarms
};
}
public void Setup()
{
_sqs = new Sqs
{
LengthThreshold = 42,
OldestMessageThreshold = 42,
Queues = new List <Queue>
{
new Queue
{
Name = "QueueName",
LengthThreshold = 42,
OldestMessageThreshold = 42,
Errors = new ErrorQueue
{
LengthThreshold = 42,
OldestMessageThreshold = 42
}
}
},
Errors = new ErrorQueue
{
LengthThreshold = 42,
OldestMessageThreshold = 42
}
};
_config = ConfigTestData.ValidConfig();
_config.AlertingGroups.First().Sqs = _sqs;
}
public async Task TestNullRefIssueWhenMixingOldAndNewConfig()
{
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "ag1",
AlarmNameSuffix = "ag1suffix",
Services = null,
DynamoDb = new DynamoDb()
{
Tables = new List <Table>()
{
new Table()
{
Name = "banana"
}
}
}
},
new AlertingGroup()
{
Name = "ag2",
AlarmNameSuffix = "ag2suffix",
Services = new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "asg"
}
}
}
}
}
}
};
var cloudformation = new FakeCloudFormation();
var context = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
context.GetMock <IAmazonAutoScaling>()
.HasAutoScalingGroups(new AutoScalingGroup[0]);
await context.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// pass as did not throw
}
public void Config_Fails_When_Null()
{
// arrange
_config = null;
// act
// assert
ConfigAssert.NotValid(_config, "Config cannot be null");
}
public static IConfigLoader ConfigLoaderFor(WatchmanConfiguration config)
{
var fake = new Mock <IConfigLoader>();
fake
.Setup(x => x.LoadConfig())
.Returns(config);
return(fake.Object);
}
public void Setup()
{
var testFilePath = TestFiles.GetRelativePathTo(($"data\\withThresholds".ToCrossPlatformPath()));
var testFilesSettings = new FileSettings(testFilePath);
var logger = new Mock <IConfigLoadLogger>();
var loader = new ConfigLoader(testFilesSettings, logger.Object);
_config = loader.LoadConfig();
}
private static WatchmanServiceConfiguration <TConfig> Map <TConfig>(WatchmanConfiguration input,
string serviceName,
Func <AlertingGroup, AwsServiceAlarms <TConfig> > readServiceFromGroup) where TConfig : class
{
var groups = input.AlertingGroups
.Select(x => ServiceAlertingGroup(x, readServiceFromGroup))
.ToList();
return(new WatchmanServiceConfiguration <TConfig>(serviceName, groups));
}
public void Setup()
{
var testFilePath = TestFiles.GetRelativePathTo("data");
var testFilesSettings = new FileSettings(testFilePath);
var logger = new Mock <IConfigLoadLogger>();
var loader = new ConfigLoader(testFilesSettings, logger.Object);
_config = loader.LoadConfig();
}
private static WatchmanServiceConfiguration Map(WatchmanConfiguration input,
string serviceName,
Func <AlertingGroup, AwsServiceAlarms> readServiceFromGroup,
IList <AlarmDefinition> defaults)
{
var groups = input.AlertingGroups
.Select(x => ServiceAlertingGroup(x, readServiceFromGroup))
.Where(x => x != null)
.ToList();
return(new WatchmanServiceConfiguration(serviceName, groups, defaults));
}
public async Task GenerateAlarmsFor(WatchmanConfiguration config, RunMode mode)
{
var dryRun = mode == RunMode.DryRun;
await LogTableNames();
foreach (var alertingGroup in config.AlertingGroups)
{
await GenerateAlarmsFor(alertingGroup, dryRun);
}
ReportPutCounts(dryRun);
}
public async Task CatchAllgroupCanBeExcluded()
{
var loader = MockTableSource(new List <string> {
"tableA", "tableB", "tableC", "tableD"
});
var reporter = new OrphanTablesFinder(loader.Object);
var config = new WatchmanConfiguration
{
AlertingGroups = new List <AlertingGroup>
{
new AlertingGroup
{
DynamoDb = new DynamoDb
{
Tables = new List <Table> {
"tableA"
}
}
},
new AlertingGroup
{
DynamoDb = new DynamoDb
{
Tables = new List <Table> {
"tableD"
}
}
},
new AlertingGroup
{
Name = "catchAll",
IsCatchAll = true,
DynamoDb = new DynamoDb
{
Tables = new List <Table> {
"tableA", "tableB", "tableC", "tableD"
}
}
}
}
};
var orphans = await reporter.FindOrphanTables(config);
AssertHasOrphans(orphans, new List <string> {
"tableB", "tableC"
});
}
public async Task GenerateAlarmsForService(
WatchmanConfiguration config, RunMode mode)
{
var serviceConfig = _serviceConfigMapper(config);
if (!ServiceConfigIsPopulated(serviceConfig))
{
_logger.Info($"No resources for {serviceConfig.ServiceName}. No action taken for this resource type");
return;
}
await PopulateResourceNames(serviceConfig);
await GenerateAlarms(serviceConfig, mode);
await ReportOrphans(serviceConfig);
}
public static void Validate(WatchmanConfiguration config)
{
if (config == null)
{
throw new ConfigException("Config cannot be null");
}
if (!HasAny(config.AlertingGroups))
{
throw new ConfigException("Config must have alerting groups");
}
foreach (var alertingGroup in config.AlertingGroups)
{
Validate(alertingGroup);
}
}
public void Setup()
{
_dynamoDb = new DynamoDb
{
Threshold = 0.7,
ThrottlingThreshold = 42,
Tables = new List <Table>
{
new Table
{
Name = "TableName",
Threshold = 0.8
}
}
};
_config = ConfigTestData.ValidConfig();
_config.AlertingGroups.First().DynamoDb = _dynamoDb;
}
private async Task GenerateAlarms(WatchmanConfiguration config, RunMode mode)
{
await _dynamoGenerator.GenerateAlarmsFor(config, mode);
await _orphanTablesReporter.FindAndReport(config);
await _sqsGenerator.GenerateAlarmsFor(config, mode);
await _orphanQueuesReporter.FindAndReport(config);
foreach (var service in _otherServices)
{
await service.GenerateAlarmsForService(config, mode);
}
if (mode == RunMode.DryRun || mode == RunMode.GenerateAlarms)
{
await _creator.SaveChanges(mode == RunMode.DryRun);
}
}
public async Task <OrphansModel> FindOrphanTables(WatchmanConfiguration config)
{
var monitoredTables = config.AlertingGroups
.Where(ag => !ag.IsCatchAll && ag.DynamoDb?.Tables != null)
.SelectMany(ag => ag.DynamoDb.Tables)
.Select(t => t.Name)
.Distinct();
var allTables = await _tableSource.GetResourceNamesAsync();
var unMonitoredTables = allTables
.Except(monitoredTables)
.OrderBy(t => t)
.ToList();
return(new OrphansModel
{
Items = unMonitoredTables.ToList(),
ServiceName = "table"
});
}
public async Task NamedTableThatDoesNotExistIsIgnored()
{
var mockery = new DynamoAlarmGeneratorMockery();
var generator = mockery.AlarmGenerator;
ConfigureTables(mockery);
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "TestGroup",
AlarmNameSuffix = "TestGroup",
DynamoDb = new DynamoDb()
{
Tables = new List <Table>
{
new Table {
Name = "table-that-exists"
},
new Table {
Name = "missing-table"
}
}
}
}
}
};
await generator.GenerateAlarmsFor(config, RunMode.GenerateAlarms);
CloudwatchVerify.AlarmWasPutOnTable(mockery.Cloudwatch,
alarmName: "table-that-exists-ConsumedWriteCapacityUnits-TestGroup",
tableName: "table-that-exists",
metricName: "ConsumedWriteCapacityUnits",
threshold: 144000,
period: 300);
}
public async Task MultipleAlertingGroupsAreCovered()
{
var loader = MockTableSource(new List <string> {
"tableA", "tableB", "tableC", "tableD"
});
var reporter = new OrphanTablesFinder(loader.Object);
var config = new WatchmanConfiguration
{
AlertingGroups = new List <AlertingGroup>
{
new AlertingGroup
{
DynamoDb = new DynamoDb
{
Tables = new List <Table> {
"tableA"
}
}
},
new AlertingGroup
{
DynamoDb = new DynamoDb
{
Tables = new List <Table> {
"tableD"
}
}
}
}
};
var orphans = await reporter.FindOrphanTables(config);
AssertHasOrphans(orphans, new List <string> {
"tableB", "tableC"
});
}
public async Task CanOptOut()
{
// arrange
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "group-with-description",
AlarmNameSuffix = "group-suffix-1",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
// to test we can opt out at resource level
{ "LatencyHigh", new AlarmValues(enabled: false) },
// to test we can revert an opt-out at service level
{ "UnHealthyHostCountHigh", new AlarmValues(enabled: true) }
}
}
},
Values = new Dictionary <string, AlarmValues>()
{
// test we can opt out at service level
{ "Http5xxErrorsHigh", new AlarmValues(enabled: false) },
// setup for above (we can revert an opt-out at service level)
{ "UnHealthyHostCountHigh", new AlarmValues(enabled: false) }
}
}
}
}
}
};
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = fakeCloudFormation
.Stack("Watchman-group-with-description")
.AlarmsByDimension("LoadBalancerName");
var alarmsForElb1 = alarms["elb-1"];
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("LatencyHigh")
), Is.False);
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("Http5xxErrorsHigh")
), Is.False);
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("UnHealthyHostCountHigh")
), Is.True);
}
public static WatchmanServiceConfiguration <ResourceConfig> MapStepFunction(WatchmanConfiguration input)
{
const string id = "StepFunction";
return(Map(input, id, a => a?.Services?.StepFunction, Defaults.StepFunction));
}
public static WatchmanServiceConfiguration <ResourceConfig> MapDynamoDb(WatchmanConfiguration input)
{
const string id = "DynamoDb";
return(Map(input, id, a => a?.Services?.DynamoDb, Defaults.DynamoDb));
}
public async Task FindAndReport(WatchmanConfiguration config)
{
var orphans = await _orphanTablesFinder.FindOrphanTables(config);
_logger.Log(orphans);
}
public static void HasConfig(this Mock <IConfigLoader> loader, WatchmanConfiguration config)
{
loader.Setup(x => x.LoadConfig()).Returns(config);
}
public async Task AlarmsHaveMeaningfulDescription()
{
// arrange
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "group-with-description",
AlarmNameSuffix = "group-suffix-1",
Description = "Group description for group 1",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1"
}
}
}
}
},
new AlertingGroup()
{
Name = "group-without-description",
AlarmNameSuffix = "group-suffix-2",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-2"
}
}
}
}
}
}
};
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var firstGroupAlarm = fakeCloudFormation
.Stack("Watchman-group-with-description")
.Alarms()
.First();
var description = firstGroupAlarm.Properties["AlarmDescription"].ToString();
Assert.That(description, Contains.Substring("Alarm generated by AwsWatchman"));
Assert.That(description, Contains.Substring("group-with-description"));
Assert.That(description, Contains.Substring("Group description for group 1"));
var secondGroupAlarm = fakeCloudFormation
.Stack("Watchman-group-without-description")
.Alarms()
.First();
var description2 = secondGroupAlarm.Properties["AlarmDescription"].ToString();
Assert.That(description2, Contains.Substring("Alarm generated by AwsWatchman"));
Assert.That(description2, Contains.Substring("group-without-description"));
}
