private async Task <List <ResourceThresholds> > ExpandTablePatterns(AwsServiceAlarms service, string alertingGroupName)
{
var named = service.Resources
.Where(t => string.IsNullOrWhiteSpace(t.Pattern))
.ToList();
var patterns = service.Resources
.Where(t => !string.IsNullOrWhiteSpace(t.Pattern))
.ToList();
var matchedPatterns = new List <ResourceThresholds>();
foreach (var pattern in patterns)
{
var matches = await GetPatternMatches(pattern, alertingGroupName);
matchedPatterns.AddRange(matches);
}
var all = Distinct(named.Union(matchedPatterns));
if (service.ExcludeResourcesPrefixedWith == null)
{
return(all.ToList());
}
return(all.Where(
a => !service.ExcludeResourcesPrefixedWith.Any(prefix => a.Name.StartsWith(prefix))
).ToList());
}
public void Setup()
{
_awsServiceAlarms = new AwsServiceAlarms <ResourceConfig>
{
ExcludeResourcesPrefixedWith = new List <string>
{
"ExcludePrefix"
},
Resources = new List <ResourceThresholds <ResourceConfig> >
{
new ResourceThresholds <ResourceConfig>
{
Name = "ResourceName",
Values = new Dictionary <string, AlarmValues>
{
{ "testThresholdLow", 42 }
}
}
},
Values = new Dictionary <string, AlarmValues>
{
{ "testThresholdHigh", 242 }
}
};
_config = ConfigTestData.ValidConfig();
_config.AlertingGroups.First().Services = new AlertingGroupServices()
{
Lambda = _awsServiceAlarms
};
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
IList <AlarmDefinition> defaults,
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
string groupSuffix)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
// apply thresholds from resource or alerting group
var expanded = await _builder.CopyAndUpdateDefaultAlarmsForResource(entity, defaults, service, resource);
var result = new List <Alarm>();
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
foreach (var alarm in expanded)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = _builder.GetAlarmName(entity, alarm.Name, groupSuffix),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = alarm
};
result.Add(model);
}
return(result);
}
private async Task <AlarmDefinition> AlarmWithMergedValues(
AwsResource <T> entity,
AlarmDefinition alarm,
AwsServiceAlarms <TAlarmConfig> serviceConfig,
ResourceThresholds <TAlarmConfig> resourceConfig)
{
var mergedThreshold = MergeValueOverrides(alarm.Name, new [] { resourceConfig.Values, serviceConfig.Values });
var config = MergeConfiguration(serviceConfig.Options, resourceConfig.Options);
var copy = alarm.Copy();
copy.Threshold = await ExpandThreshold(entity.Resource, config, new Threshold
{
SourceAttribute = alarm.Threshold.SourceAttribute,
ThresholdType = alarm.Threshold.ThresholdType,
Value = mergedThreshold.Threshold ?? alarm.Threshold.Value
});
copy.EvaluationPeriods = mergedThreshold.EvaluationPeriods ?? alarm.EvaluationPeriods;
copy.ExtendedStatistic = !string.IsNullOrEmpty(mergedThreshold.ExtendedStatistic)
? mergedThreshold.ExtendedStatistic
: alarm.ExtendedStatistic;
return(copy);
}
private static void AssertSectionIsPopulated(AwsServiceAlarms section)
{
Assert.That(section, Is.Not.Null);
Assert.That(section.ExcludeResourcesPrefixedWith, Is.Not.Null);
Assert.That(section.ExcludeResourcesPrefixedWith, Is.Not.Empty);
Assert.That(section.Resources, Is.Not.Null);
Assert.That(section.Resources, Is.Not.Empty);
}
private async Task <IList <Alarm> > BuildIndexAlarms(ResourceThresholds <DynamoResourceConfig> resourceConfig,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> parentTableEntity)
{
// called twice
var mergedConfig = service.Options.OverrideWith(resourceConfig.Options);
var result = new List <Alarm>();
var gsiSet = parentTableEntity.Resource.GlobalSecondaryIndexes;
var mergedValuesByAlarmName = service.Values.OverrideWith(resourceConfig.Values);
var defaults = _defaultAlarms.DynamoDbGsiRead;
if (mergedConfig.MonitorWrites ?? DynamoResourceConfig.MonitorWritesDefault)
{
defaults = defaults.Concat(_defaultAlarms.DynamoDbGsiWrite).ToArray();
}
foreach (var gsi in gsiSet)
{
var gsiResource = new AwsResource <GlobalSecondaryIndexDescription>(gsi.IndexName, gsi);
foreach (var alarm in defaults)
{
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var dimensions = _gsiProvider.GetDimensions(gsi, parentTableEntity.Resource, alarm.DimensionNames);
var threshold = await ThresholdCalculator.ExpandThreshold(_gsiProvider,
gsiResource.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resourceConfig.Name}-{gsi.IndexName}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
// TODO: remove this property in a future PR
// passing in the entity shouldn't be necessary and passing in the table entity here
// when the alarm is for a GSI is an even worse hack
Resource = parentTableEntity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
}
return(result);
}
private static ServiceAlertingGroup Map(AlertingGroup input, AwsServiceAlarms service)
{
return(new ServiceAlertingGroup
{
AlarmNameSuffix = input.AlarmNameSuffix,
IsCatchAll = input.IsCatchAll,
Name = input.Name,
ReportTargets = input.ReportTargets,
Service = service,
Targets = input.Targets
});
}
private static ServiceAlertingGroup <T> Map <T>(AlertingGroup input, AwsServiceAlarms <T> service) where T : class
{
return(new ServiceAlertingGroup <T>
{
GroupParameters = new AlertingGroupParameters(
input.Name,
input.AlarmNameSuffix,
input.Targets,
input.IsCatchAll
),
Service = service
});
}
private async Task <IList <Alarm> > ExpandAlarmsToResources(
IList <AlarmDefinition> defaults,
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
string groupSuffix)
{
// apply thresholds from resource or alerting group
var expanded = ExpandDefaultAlarmsForResource(defaults, resource.Values, service.Values);
var config = MergeConfiguration(service.Options, resource.Options);
return(await GetAlarms(expanded, resource, config, groupSuffix));
}
private async Task <List <Alarm> > BuildTableAlarms(ResourceThresholds <DynamoResourceConfig> resourceConfig,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var mergedConfig = service.Options.OverrideWith(resourceConfig.Options);
var result = new List <Alarm>();
var mergedValuesByAlarmName = service.Values.OverrideWith(resourceConfig.Values);
var defaults = _defaultAlarms.DynamoDbRead;
if (mergedConfig.MonitorWrites ?? DynamoResourceConfig.MonitorWritesDefault)
{
defaults = defaults.Concat(_defaultAlarms.DynamoDbWrite).ToArray();
}
foreach (var alarm in defaults)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
if (mergedConfig.ThresholdIsAbsolute ?? DynamoResourceConfig.ThresholdIsAbsoluteDefault)
{
configuredThreshold.ThresholdType = ThresholdType.Absolute;
}
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
entity.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resourceConfig.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
return(result);
}
private static ServiceAlertingGroup <T> Map <T>(AlertingGroup input, AwsServiceAlarms <T> service) where T : class
{
return(new ServiceAlertingGroup <T>
{
GroupParameters = new AlertingGroupParameters(
input.Name,
input.AlarmNameSuffix,
input.Targets,
input.IsCatchAll,
input.Description,
input.NumberOfCloudFormationStacks
),
Service = service
});
}
public static void Validate(string alertingGroupName, string serviceName, AwsServiceAlarms serviceAlarms)
{
if (serviceAlarms.Values != null)
{
foreach (var threshold in serviceAlarms.Values)
{
ValidServiceThreshold(threshold);
}
}
foreach (var resource in serviceAlarms.Resources)
{
ValidateResource(alertingGroupName, serviceName, resource);
}
}
private async Task <List <Alarm> > BuildAlarmsForQueue(
IList <AlarmDefinition> defaults,
ResourceThresholds <SqsResourceConfig> resource,
AwsServiceAlarms <SqsResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <QueueDataV2> entity)
{
var mergedConfig = service.Options.OverrideWith(resource.Options);
bool includeErrorQueues = mergedConfig.IncludeErrorQueues ?? true;
var result = new List <Alarm>();
var mergedValuesByAlarmName = service.Values.OverrideWith(resource.Values);
foreach (var alarm in defaults)
{
var dimensions = _dimensionProvider.GetDimensions(entity.Resource, alarm.DimensionNames);
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var actualThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
entity.Resource,
mergedConfig,
actualThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
if (includeErrorQueues && entity.Resource.ErrorQueue != null)
{
var errorQueueResource = new AwsResource <QueueDataV2>(entity.Name, entity.Resource.ErrorQueue);
result.AddRange(await BuildAlarmsForQueue(_errorQueueDefaults, resource, service, groupParameters, errorQueueResource));
}
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <SqsResourceConfig> resource,
AwsServiceAlarms <SqsResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _queueSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var queueResource = new AwsResource <QueueDataV2>(entity.Name, entity.Resource);
var alarms = await BuildAlarmsForQueue(_defaultAlarms, resource, service, groupParameters, queueResource);
return(alarms);
}
public async Task <IList <AlarmDefinition> > CopyAndUpdateDefaultAlarmsForResource(
AwsResource <T> entity,
IList <AlarmDefinition> alerts,
AwsServiceAlarms <TAlarmConfig> serviceConfig,
ResourceThresholds <TAlarmConfig> resourceConfig)
{
var result = new List <AlarmDefinition>();
foreach (var defaultAlarm in alerts)
{
var alarm = await AlarmWithMergedValues(entity, defaultAlarm, serviceConfig, resourceConfig);
result.Add(alarm);
}
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <DynamoResourceConfig> resource,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var result = await BuildTableAlarms(resource, service, groupParameters, entity);
result.AddRange(await BuildIndexAlarms(resource, service, groupParameters, entity));
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var mergedConfig = service.Options.OverrideWith(resource.Options);
var mergedValuesByAlarmName = service.Values.OverrideWith(resource.Values);
var result = new List <Alarm>();
foreach (var alarm in _defaultAlarms)
{
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
entity.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <DynamoResourceConfig> resource,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
_logger.Error($"Skipping table {resource.Name} as it does not exist");
return(Array.Empty <Alarm>());
}
var result = await BuildTableAlarms(resource, service, groupParameters, entity);
result.AddRange(await BuildIndexAlarms(resource, service, groupParameters, entity));
return(result);
}
public async Task <IList <Alarm> > GenerateAlarmsFor(
AwsServiceAlarms <TAlarmConfig> service,
IList <AlarmDefinition> defaults,
string alarmSuffix)
{
if (service?.Resources == null || service.Resources.Count == 0)
{
return(new List <Alarm>());
}
var allAlarms = await Task.WhenAll(service.Resources
.Select(r => ExpandAlarmsToResources(
defaults,
r,
service,
alarmSuffix)));
return(allAlarms.SelectMany(x => x).ToList());
}
public async Task <IList <Alarm> > GenerateAlarmsFor(
AwsServiceAlarms <TAlarmConfig> service,
AlertingGroupParameters groupParameters)
{
if (service?.Resources == null || service.Resources.Count == 0)
{
return(new List <Alarm>());
}
List <Alarm> alarms = new List <Alarm>();
foreach (var resource in service.Resources)
{
var alarmsForResource = await CreateAlarmsForResource(resource, service, groupParameters);
alarms.AddRange(alarmsForResource);
}
return(alarms);
}
public async Task <IList <Alarm> > GenerateAlarmsFor(
AwsServiceAlarms <TAlarmConfig> service,
IList <AlarmDefinition> defaults,
string alarmSuffix)
{
if (service?.Resources == null || service.Resources.Count == 0)
{
return(new List <Alarm>());
}
List <Alarm> alarms = new List <Alarm>();
foreach (var resource in service.Resources)
{
var alarmsForResource = await CreateAlarmsForResource(defaults, resource, service, alarmSuffix);
alarms.AddRange(alarmsForResource);
}
return(alarms);
}
private async Task <List <ResourceAndThresholdsPair <TConfig, T> > > ExpandTablePatterns(
AwsServiceAlarms <TConfig> service,
string alertingGroupName
)
{
var named = service.Resources
.Where(t => string.IsNullOrWhiteSpace(t.Pattern))
.Select(t => t.AsPattern())
.ToList();
var patterns = service.Resources
.Where(t => !string.IsNullOrWhiteSpace(t.Pattern))
.ToList();
var matchedPatterns = new List <ResourceAndThresholdsPair <TConfig, T> >();
var combined = named.Concat(patterns);
foreach (var pattern in combined)
{
var matches = await GetPatternMatches(pattern, alertingGroupName);
matchedPatterns.AddRange(matches);
}
var all = Distinct(matchedPatterns);
if (service.ExcludeResourcesPrefixedWith == null)
{
return(all.ToList());
}
return(all.Where(
a => !service.ExcludeResourcesPrefixedWith
.Any(prefix => a.Resource.Name.StartsWith(prefix))
).ToList());
}
private async Task <List <Alarm> > BuildTableAlarms(IList <AlarmDefinition> defaults, ResourceThresholds <ResourceConfig> resource, AwsServiceAlarms <ResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var expanded = await _builder.CopyAndUpdateDefaultAlarmsForResource(entity, defaults, service, resource);
var result = new List <Alarm>();
foreach (var alarm in expanded)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = _builder.GetAlarmDescription(groupParameters),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = alarm
};
result.Add(model);
}
return(result);
}
private async Task <IList <Alarm> > BuildIndexAlarms(ResourceThresholds <ResourceConfig> resource, AwsServiceAlarms <ResourceConfig> service, AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var result = new List <Alarm>();
var gsiSet = entity.Resource.GlobalSecondaryIndexes;
foreach (var gsi in gsiSet)
{
var gsiResource = new AwsResource <GlobalSecondaryIndexDescription>(gsi.IndexName, gsi);
var expandedGsi = await _gsiAlarmBuilder.CopyAndUpdateDefaultAlarmsForResource(
gsiResource,
Defaults.DynamoDbGsi, service, resource);
foreach (var gsiAlarm in expandedGsi)
{
var dimensions = _gsiDimensionProvider.GetDimensions(gsi, gsiAlarm.DimensionNames);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{gsi.IndexName}-{gsiAlarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = _builder.GetAlarmDescription(groupParameters),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = gsiAlarm
};
result.Add(model);
}
}
return(result);
}
private async Task <IList <Alarm> > ExpandAlarmsToResources(ServiceAlertingGroup alertingGroup,
IList <AlarmDefinition> defaults,
ResourceThresholds resource, AwsServiceAlarms service)
{
// apply thresholds from resource or alerting group
var expanded = ExpandDefaultAlarmsForResource(defaults, resource.Values, service.Values);
return(await GetAlarms(expanded, resource, alertingGroup));
}
