private async Task GenerateAlarmsFor(AlertingGroup alertingGroup, bool dryRun)
{
if (alertingGroup.DynamoDb?.Tables == null || alertingGroup.DynamoDb.Tables.Count == 0)
{
return;
}
await _populator.PopulateDynamoTableNames(alertingGroup);
var snsTopic = await _snsCreator.EnsureSnsTopic(alertingGroup, dryRun);
var readAlarms = AlarmTablesHelper.FilterForRead(alertingGroup);
readAlarms.SnsTopicArn = snsTopic;
readAlarms.DryRun = dryRun;
readAlarms.ThrottlingThreshold = alertingGroup.DynamoDb.ThrottlingThreshold ?? AwsConstants.ThrottlingThreshold;
await EnsureReadAlarms(readAlarms);
var writeAlarms = AlarmTablesHelper.FilterForWrite(alertingGroup);
writeAlarms.SnsTopicArn = snsTopic;
writeAlarms.DryRun = dryRun;
writeAlarms.ThrottlingThreshold = alertingGroup.DynamoDb.ThrottlingThreshold ?? AwsConstants.ThrottlingThreshold;
await EnsureWriteAlarms(writeAlarms);
}
private void LogAlertingGroup(string configFileName, AlertingGroup group)
{
var containedServiceCounts = CountContainedServices(group);
_logger.Info($"Read alerting group {group.Name} containing {containedServiceCounts} from file {configFileName}");
if (group.IsCatchAll)
{
_logger.Detail($"Alerting group {group.Name} is catch-all");
}
var monitorThrottling = group.DynamoDb.MonitorThrottling ?? true;
if (monitorThrottling)
{
_logger.Detail($"Alerting group {group.Name} is monitoring throttled reads and writes");
}
if (group.DynamoDb?.Tables != null)
{
foreach (var table in group.DynamoDb.Tables)
{
_logger.Detail(DescribeTable(table));
}
}
if (group.Sqs?.Queues != null)
{
foreach (var queue in group.Sqs.Queues)
{
_logger.Detail(DescribeQueue(queue));
}
}
}
private static string CountContainedServices(AlertingGroup group)
{
var tableCount = group.DynamoDb.Tables?.Count ?? 0;
var queueCount = group.Sqs.Queues?.Count ?? 0;
var serviceCount = group.Services?.AllServices?.Count ?? 0;
if ((tableCount == 0) && (queueCount == 0) && (serviceCount == 0))
{
return("nothing");
}
var items = new List <string>();
if (tableCount > 0)
{
items.Add($"{tableCount} tables");
}
if (queueCount > 0)
{
items.Add($"{queueCount} queues");
}
if (serviceCount > 0)
{
items.Add($"{serviceCount} service definitions");
}
return(string.Join(", ", items));
}
public async Task MatchAllTablesGetsTablesFromTableLoader()
{
var alertingGroup = new AlertingGroup
{
Name = "test",
AlarmNameSuffix = "test",
DynamoDb = new DynamoDb
{
Tables = new List <Table>
{
new Table {
Pattern = ".*"
}
}
}
};
var populator = CreatePopulator(true);
await populator.PopulateDynamoTableNames(alertingGroup);
Assert.That(alertingGroup.DynamoDb.Tables.Count, Is.EqualTo(3));
ShouldHaveTable(alertingGroup.DynamoDb.Tables, "AutoTable1");
ShouldHaveTable(alertingGroup.DynamoDb.Tables, "AutoTable2");
ShouldHaveTable(alertingGroup.DynamoDb.Tables, "ATable3");
_tableLoaderMock.Verify(t => t.GetResourceNamesAsync(), Times.Once);
}
public void ReadFilterIsAppliedForReadOnly()
{
var input = new AlertingGroup
{
DynamoDb = new DynamoDb
{
Tables = new List <Table> {
"table1", "table2", "not_this_one"
},
ExcludeTablesPrefixedWith = new List <string> {
"not"
},
ExcludeReadsForTablesPrefixedWith = new List <string> {
"table1"
},
ExcludeWritesForTablesPrefixedWith = new List <string> {
"nomatch"
}
}
};
var readTables = AlarmTablesHelper.FilterForRead(input);
var writeTables = AlarmTablesHelper.FilterForWrite(input);
Assert.That(readTables.Tables, Is.EquivalentTo(new List <Table> {
"table2"
}));
Assert.That(writeTables.Tables, Is.EquivalentTo(new List <Table> {
"table1", "table2"
}));
}
public async Task TableListIsPassedThrough()
{
var alertingGroup = new AlertingGroup
{
Name = "test",
AlarmNameSuffix = "test",
DynamoDb = new DynamoDb
{
Tables = new List <Table>
{
new Table {
Name = "tableA", Threshold = 0.5
},
new Table {
Name = "tableB", Threshold = 0.4
}
}
}
};
var populator = CreatePopulator(false);
await populator.PopulateDynamoTableNames(alertingGroup);
Assert.That(alertingGroup.DynamoDb.Tables.Count, Is.EqualTo(2));
ShouldHaveTable(alertingGroup.DynamoDb.Tables, "tableA");
ShouldHaveTable(alertingGroup.DynamoDb.Tables, "tableB");
_tableLoaderMock.Verify(t => t.GetResourceNamesAsync(), Times.Never);
}
private static WatchmanConfiguration MakePatternConfig()
{
var alertingGroup = new AlertingGroup
{
AlarmNameSuffix = "test1",
Sqs = new Configuration.Sqs
{
Queues = new List <Queue>
{
new Queue
{
Pattern = "pattern",
LengthThreshold = 10,
Errors = new ErrorQueue
{
LengthThreshold = 1
}
}
}
}
};
return(new WatchmanConfiguration
{
AlertingGroups = new List <AlertingGroup>
{
alertingGroup
}
});
}
private static void Validate(AlertingGroup alertingGroup)
{
if (string.IsNullOrWhiteSpace(alertingGroup.Name))
{
throw new ConfigException("AlertingGroup must have a name");
}
if (!TextIsValidInSnsTopic(alertingGroup.Name))
{
throw new ConfigException($"AlertingGroup name '{alertingGroup.Name}' must be valid in SNS topics");
}
if (string.IsNullOrWhiteSpace(alertingGroup.AlarmNameSuffix))
{
throw new ConfigException($"AlertingGroup '{alertingGroup.Name}' must have an alarm suffix");
}
if (!TextIsValidInSnsTopic(alertingGroup.AlarmNameSuffix))
{
throw new ConfigException($"AlertingGroup '{alertingGroup.Name}' must have a suffix valid in SNS topics. '{alertingGroup.AlarmNameSuffix}' is not.");
}
ValidateTargets(alertingGroup);
var hasAtLeastOneResource = false;
if (HasAny(alertingGroup.DynamoDb?.Tables))
{
hasAtLeastOneResource = true;
DynamoDbValidation.Validate(alertingGroup.Name, alertingGroup.DynamoDb);
}
if (HasAny(alertingGroup.Sqs?.Queues))
{
hasAtLeastOneResource = true;
SqsValidation.Validate(alertingGroup.Name, alertingGroup.Sqs);
}
if (alertingGroup.Services != null)
{
foreach (var service in alertingGroup.Services.AllServicesByName)
{
if (service.Value != null)
{
AwsServiceValidation.Validate(alertingGroup.Name, service.Key, service.Value);
if (HasAny(service.Value.Resources))
{
hasAtLeastOneResource = true;
}
}
}
}
if (!hasAtLeastOneResource)
{
throw new ConfigException($"AlertingGroup '{alertingGroup.Name}' must contain resources to monitor. " +
"Specify one or more of DynamoDb, Sqs or other resources");
}
}
private void ReadIntoTargets(AlertingGroup result, JToken jToken)
{
if (!jToken.HasValues)
{
return;
}
foreach (var item in jToken.Children())
{
if (item["Email"] != null)
{
result.Targets.Add(new AlertEmail {
Email = item["Email"].ToString()
});
}
else if (item["Url"] != null)
{
result.Targets.Add(new AlertUrl {
Url = item["Url"].ToString()
});
}
else
{
_logger.Warn($"The target {jToken} is unknown. Valid targets are 'Email' and 'Url'.");
}
}
}
private static void SetErrorDefaultsOnQueue(AlertingGroup group, Queue queue)
{
if (queue.Errors == null)
{
queue.Errors = new ErrorQueue();
}
queue.Errors.ReadDefaults(group.Sqs.Errors);
}
private static void SetErrorDefaultsOnAlertingGroup(AlertingGroup alertingGroup)
{
if (alertingGroup.Sqs.Errors == null)
{
alertingGroup.Sqs.Errors = new ErrorQueue();
}
alertingGroup.Sqs.Errors.ReadDefaults(ErrorQueueDefaults);
}
private static void SetErrorDefaults(AlertingGroup alertingGroup)
{
SetErrorDefaultsOnAlertingGroup(alertingGroup);
foreach (var configuredQueue in alertingGroup.Sqs.Queues)
{
SetErrorDefaultsOnQueue(alertingGroup, configuredQueue);
}
}
public static WatchmanConfiguration WrapGroup(AlertingGroup group)
{
return(new WatchmanConfiguration
{
AlertingGroups = new List <AlertingGroup> {
group
}
});
}
private WatchmanConfiguration MakeConfigFor(AlertingGroup group)
{
return(new WatchmanConfiguration
{
AlertingGroups = new List <AlertingGroup> {
group
}
});
}
private int QueueLengthThreshold(Queue queue, AlertingGroup group)
{
if (queue.IsErrorQueue())
{
return(queue.Errors.LengthThreshold.Value);
}
return(queue.LengthThreshold ?? group.Sqs.LengthThreshold ?? AwsConstants.QueueLengthThreshold);
}
private int?OldestMessageThreshold(Queue queue, AlertingGroup group)
{
if (queue.IsErrorQueue())
{
return(queue.Errors.OldestMessageThreshold);
}
return(queue.OldestMessageThreshold ?? group.Sqs.OldestMessageThreshold ?? AwsConstants.OldestMessageThreshold);
}
private static AwsServiceAlarms GetService(AlertingGroup group, string serviceIdentifier)
{
if (!group.Services.ContainsKey(serviceIdentifier))
{
return(null);
}
return(group.Services[serviceIdentifier]);
}
private static void AssertGroupDataIsLoaded(AlertingGroup alertingGroup)
{
Assert.That(alertingGroup.Name, Is.Not.Empty);
Assert.That(alertingGroup.Targets, Is.Not.Empty);
Assert.That(alertingGroup.AlarmNameSuffix, Is.Not.Empty);
var alertEmail = (AlertEmail)alertingGroup.Targets.First();
Assert.That(alertEmail.Email, Is.Not.Empty);
}
public async Task <string> EnsureSnsTopic(AlertingGroup alertingGroup, bool dryRun)
{
var snsTopicArn = await _snsTopicCreator.EnsureSnsTopic(alertingGroup.Name, dryRun);
if (!dryRun)
{
await _snsSubscriptionCreator.EnsureSnsSubscriptions(alertingGroup, snsTopicArn);
}
return(snsTopicArn);
}
private static int CountGenericServices(AlertingGroup group)
{
if (group.Services == null)
{
return(0);
}
return(group.Services.Values
.Select(v => v.Resources?.Count ?? 0)
.Sum());
}
private static ServiceAlertingGroup ServiceAlertingGroup(AlertingGroup ag, Func <AlertingGroup, AwsServiceAlarms> readServiceFromGroup)
{
var service = readServiceFromGroup(ag);
if (service == null)
{
return(null);
}
return(Map(ag, service));
}
private static void ReadServiceDefinitions(JObject jsonObject, AlertingGroup result, JsonSerializer serializer)
{
if (jsonObject["DynamoDb"] != null)
{
result.DynamoDb = jsonObject["DynamoDb"].ToObject <DynamoDb>(serializer);
}
if (jsonObject["Sqs"] != null)
{
result.Sqs = jsonObject["Sqs"].ToObject <Sqs>(serializer);
}
}
private static ServiceAlertingGroup Map(AlertingGroup input, AwsServiceAlarms service)
{
return(new ServiceAlertingGroup
{
AlarmNameSuffix = input.AlarmNameSuffix,
IsCatchAll = input.IsCatchAll,
Name = input.Name,
ReportTargets = input.ReportTargets,
Service = service,
Targets = input.Targets
});
}
private static ServiceAlertingGroup <T> ServiceAlertingGroup <T>(AlertingGroup ag,
Func <AlertingGroup, AwsServiceAlarms <T> > readServiceFromGroup) where T : class
{
var service = readServiceFromGroup(ag);
if (service == null)
{
return(null);
}
return(Map <T>(ag, service));
}
private static ServiceAlertingGroup <T> Map <T>(AlertingGroup input, AwsServiceAlarms <T> service) where T : class
{
return(new ServiceAlertingGroup <T>
{
GroupParameters = new AlertingGroupParameters(
input.Name,
input.AlarmNameSuffix,
input.Targets,
input.IsCatchAll
),
Service = service
});
}
public static AlarmTables FilterForRead(AlertingGroup alertingGroup)
{
var filteredTables = alertingGroup.DynamoDb.Tables
.ExcludePrefixes(alertingGroup.DynamoDb.ExcludeTablesPrefixedWith, t => t.Name)
.ExcludePrefixes(alertingGroup.DynamoDb.ExcludeReadsForTablesPrefixedWith, t => t.Name);
return(new AlarmTables
{
AlarmNameSuffix = alertingGroup.AlarmNameSuffix,
Threshold = alertingGroup.DynamoDb.Threshold ?? AwsConstants.DefaultCapacityThreshold,
MonitorThrottling = alertingGroup.DynamoDb.MonitorThrottling ?? true,
Tables = filteredTables.ToList()
});
}
private static ServiceAlertingGroup <T> ServiceAlertingGroup <T>(AlertingGroup ag,
Func <AlertingGroup, AwsServiceAlarms <T> > readServiceFromGroup) where T : class
{
var service = readServiceFromGroup(ag);
if (service == null)
{
return(Map <T>(ag, new AwsServiceAlarms <T>()
{
Resources = new List <ResourceThresholds <T> >()
}));
}
return(Map <T>(ag, service));
}
private static ServiceAlertingGroup <T> Map <T>(AlertingGroup input, AwsServiceAlarms <T> service) where T : class
{
return(new ServiceAlertingGroup <T>
{
GroupParameters = new AlertingGroupParameters(
input.Name,
input.AlarmNameSuffix,
input.Targets,
input.IsCatchAll,
input.Description,
input.NumberOfCloudFormationStacks
),
Service = service
});
}
private static void ReadServiceDefinitions(JObject jsonObject, AlertingGroup result, JsonSerializer serializer)
{
var readDynamo = false;
var readSqs = false;
if (jsonObject["DynamoDb"] != null)
{
result.DynamoDb = jsonObject["DynamoDb"].ToObject <DynamoDb>(serializer);
readDynamo = true;
}
if (jsonObject["Sqs"] != null)
{
result.Sqs = jsonObject["Sqs"].ToObject <Sqs>(serializer);
readSqs = true;
}
var allServices = (JObject)jsonObject["Services"];
if (allServices != null)
{
foreach (var prop in allServices)
{
if (prop.Key == "DynamoDb")
{
if (readDynamo)
{
throw new JsonReaderException("DynamoDb block can only defined once");
}
result.DynamoDb = prop.Value.ToObject <DynamoDb>(serializer);
}
else if (prop.Key == "Sqs")
{
if (readSqs)
{
throw new JsonReaderException("Sqs block can only defined once");
}
result.Sqs = prop.Value.ToObject <Sqs>(serializer);
}
else
{
result.Services[prop.Key] = prop.Value.ToObject <AwsServiceAlarms>(serializer);
}
}
}
}
public static WatchmanConfiguration WrapDynamo(DynamoDb dynamo)
{
var ag = new AlertingGroup
{
Name = "TestGroup",
AlarmNameSuffix = "TestGroup",
IsCatchAll = true,
Targets = new List <AlertTarget>
{
new AlertEmail("*****@*****.**")
},
DynamoDb = dynamo
};
return(WrapGroup(ag));
}
