private async Task <AlarmDefinition> AlarmWithMergedValues(
AwsResource <T> entity,
AlarmDefinition alarm,
AwsServiceAlarms <TAlarmConfig> serviceConfig,
ResourceThresholds <TAlarmConfig> resourceConfig)
{
var mergedThreshold = MergeValueOverrides(alarm.Name, new [] { resourceConfig.Values, serviceConfig.Values });
var config = MergeConfiguration(serviceConfig.Options, resourceConfig.Options);
var copy = alarm.Copy();
copy.Threshold = await ExpandThreshold(entity.Resource, config, new Threshold
{
SourceAttribute = alarm.Threshold.SourceAttribute,
ThresholdType = alarm.Threshold.ThresholdType,
Value = mergedThreshold.Threshold ?? alarm.Threshold.Value
});
copy.EvaluationPeriods = mergedThreshold.EvaluationPeriods ?? alarm.EvaluationPeriods;
copy.ExtendedStatistic = !string.IsNullOrEmpty(mergedThreshold.ExtendedStatistic)
? mergedThreshold.ExtendedStatistic
: alarm.ExtendedStatistic;
return(copy);
}
private async Task <IList <Alarm> > GetAlarms(IList <AlarmDefinition> alarms,
ResourceThresholds awsResource,
ServiceAlertingGroup group)
{
var result = new List <Alarm>();
var entity = await _tableSource.GetResourceAsync(awsResource.Name);
if (entity == null)
{
throw new Exception($"Entity {awsResource.Name} not found");
}
// expand dynamic thresholds
foreach (var alarm in alarms)
{
alarm.Threshold = ExpandThreshold(entity.Resource, alarm.Threshold);
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = GetAlarmName(entity, alarm.Name, group.AlarmNameSuffix),
Resource = entity,
AlarmDefinition = alarm,
AlertingGroup = group,
Dimensions = dimensions
};
result.Add(model);
}
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
IList <AlarmDefinition> defaults,
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
string groupSuffix)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
// apply thresholds from resource or alerting group
var expanded = await _builder.CopyAndUpdateDefaultAlarmsForResource(entity, defaults, service, resource);
var result = new List <Alarm>();
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
foreach (var alarm in expanded)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = _builder.GetAlarmName(entity, alarm.Name, groupSuffix),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = alarm
};
result.Add(model);
}
return(result);
}
private async Task <IList <ResourceAndThresholdsPair <TConfig, T> > > GetPatternMatches(
ResourceThresholds <TConfig> resourcePattern,
string alertingGroupName)
{
var tableNames = await _resourceSource.GetResourcesAsync();
var regex = new Regex(resourcePattern.Pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
var matches = tableNames
.Where(table => regex.IsMatch(table.Name))
.Select(tn => PatternToTable(resourcePattern, tn))
.ToList();
if (matches.Count == 0)
{
_logger.Info($"{alertingGroupName} pattern '{resourcePattern.Pattern}' matched no resource names");
}
else if (matches.Count == tableNames.Count)
{
_logger.Info($"{alertingGroupName} pattern '{resourcePattern.Pattern}' matched all resource names");
}
else
{
_logger.Info($"{alertingGroupName} pattern '{resourcePattern.Pattern}' matched {matches.Count} of {tableNames.Count} resource names");
}
return(matches);
}
private async Task <IList <Alarm> > GetAlarms(IList <AlarmDefinition> alarms,
ResourceThresholds <TAlarmConfig> awsResource,
TAlarmConfig configuration,
string groupSuffix)
{
var result = new List <Alarm>();
var entity = await _tableSource.GetResourceAsync(awsResource.Name);
if (entity == null)
{
throw new Exception($"Entity {awsResource.Name} not found");
}
// expand dynamic thresholds
foreach (var alarm in alarms)
{
alarm.Threshold = await ExpandThreshold(entity.Resource, configuration, alarm.Threshold);
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = GetAlarmName(entity, alarm.Name, groupSuffix),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = alarm
};
result.Add(model);
}
return(result);
}
private async Task <IList <Alarm> > ExpandAlarmsToResources(ServiceAlertingGroup alertingGroup,
IList <AlarmDefinition> defaults,
ResourceThresholds resource, AwsServiceAlarms service)
{
// apply thresholds from resource or alerting group
var expanded = ExpandDefaultAlarmsForResource(defaults, resource.Values, service.Values);
return(await GetAlarms(expanded, resource, alertingGroup));
}
private static ResourceThresholds PatternToTable(ResourceThresholds pattern, string tableName)
{
return(new ResourceThresholds
{
Name = tableName,
Pattern = null,
Values = pattern.Values
});
}
private static ResourceAndThresholdsPair <TConfig, T> PatternToTable(
ResourceThresholds <TConfig> pattern, AwsResource <T> resource)
{
pattern = pattern.AsNamed(resource.Name);
var matched = new ResourceAndThresholdsPair <TConfig, T>(pattern, resource);
return(matched);
}
private static ResourceThresholds <TConfig> PatternToTable(ResourceThresholds <TConfig> pattern, string tableName)
{
return(new ResourceThresholds <TConfig>
{
Name = tableName,
Pattern = null,
Values = pattern.Values,
Options = pattern.Options
});
}
private static ResourceThresholds NameToPattern(ResourceThresholds named)
{
var name = Regex.Escape(named.Name);
return(new ResourceThresholds()
{
Pattern = $"^{name}$",
Values = named.Values
});
}
private async Task <IList <Alarm> > BuildIndexAlarms(ResourceThresholds <DynamoResourceConfig> resourceConfig,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> parentTableEntity)
{
// called twice
var mergedConfig = service.Options.OverrideWith(resourceConfig.Options);
var result = new List <Alarm>();
var gsiSet = parentTableEntity.Resource.GlobalSecondaryIndexes;
var mergedValuesByAlarmName = service.Values.OverrideWith(resourceConfig.Values);
var defaults = _defaultAlarms.DynamoDbGsiRead;
if (mergedConfig.MonitorWrites ?? DynamoResourceConfig.MonitorWritesDefault)
{
defaults = defaults.Concat(_defaultAlarms.DynamoDbGsiWrite).ToArray();
}
foreach (var gsi in gsiSet)
{
var gsiResource = new AwsResource <GlobalSecondaryIndexDescription>(gsi.IndexName, gsi);
foreach (var alarm in defaults)
{
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var dimensions = _gsiProvider.GetDimensions(gsi, parentTableEntity.Resource, alarm.DimensionNames);
var threshold = await ThresholdCalculator.ExpandThreshold(_gsiProvider,
gsiResource.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resourceConfig.Name}-{gsi.IndexName}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
// TODO: remove this property in a future PR
// passing in the entity shouldn't be necessary and passing in the table entity here
// when the alarm is for a GSI is an even worse hack
Resource = parentTableEntity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
}
return(result);
}
private static ResourceThresholds <TConfig> NameToPattern(ResourceThresholds <TConfig> named)
{
var name = Regex.Escape(named.Name);
return(new ResourceThresholds <TConfig>()
{
Pattern = $"^{name}$",
Values = named.Values,
Options = named.Options
});
}
private async Task <List <Alarm> > BuildAlarmsForQueue(
IList <AlarmDefinition> defaults,
ResourceThresholds <SqsResourceConfig> resource,
PopulatedServiceAlarms <SqsResourceConfig, QueueDataV2> service,
AlertingGroupParameters groupParameters,
QueueDataV2 queue)
{
var mergedConfig = service.Options.OverrideWith(resource.Options);
bool includeErrorQueues = mergedConfig.IncludeErrorQueues ?? true;
var result = new List <Alarm>();
var mergedValuesByAlarmName = service.Values.OverrideWith(resource.Values);
foreach (var alarm in defaults)
{
var dimensions = _dimensionProvider.GetDimensions(queue, alarm.DimensionNames);
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var actualThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
queue,
mergedConfig,
actualThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(resource),
// error queues currently named as per parent queue
ResourceIdentifier = resource.Name,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
if (includeErrorQueues && queue.ErrorQueue != null)
{
var alarms = await BuildAlarmsForQueue(_errorQueueDefaults, resource, service, groupParameters,
queue.ErrorQueue);
result.AddRange(alarms);
}
return(result);
}
private async Task <IList <Alarm> > ExpandAlarmsToResources(
IList <AlarmDefinition> defaults,
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
string groupSuffix)
{
// apply thresholds from resource or alerting group
var expanded = ExpandDefaultAlarmsForResource(defaults, resource.Values, service.Values);
var config = MergeConfiguration(service.Options, resource.Options);
return(await GetAlarms(expanded, resource, config, groupSuffix));
}
private async Task <List <Alarm> > BuildTableAlarms(ResourceThresholds <DynamoResourceConfig> resourceConfig,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var mergedConfig = service.Options.OverrideWith(resourceConfig.Options);
var result = new List <Alarm>();
var mergedValuesByAlarmName = service.Values.OverrideWith(resourceConfig.Values);
var defaults = _defaultAlarms.DynamoDbRead;
if (mergedConfig.MonitorWrites ?? DynamoResourceConfig.MonitorWritesDefault)
{
defaults = defaults.Concat(_defaultAlarms.DynamoDbWrite).ToArray();
}
foreach (var alarm in defaults)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
if (mergedConfig.ThresholdIsAbsolute ?? DynamoResourceConfig.ThresholdIsAbsoluteDefault)
{
configuredThreshold.ThresholdType = ThresholdType.Absolute;
}
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
entity.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resourceConfig.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
return(result);
}
private static ResourceAndThresholdsPair <TConfig, T> PatternToTable(
ResourceThresholds <TConfig> pattern, AwsResource <T> resource)
{
var config = new ResourceThresholds <TConfig>
{
Name = resource.Name,
Pattern = null,
Values = pattern.Values,
Options = pattern.Options
};
var matched = new ResourceAndThresholdsPair <TConfig, T>(config, resource);
return(matched);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <SqsResourceConfig> resource,
AwsServiceAlarms <SqsResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _queueSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var queueResource = new AwsResource <QueueDataV2>(entity.Name, entity.Resource);
var alarms = await BuildAlarmsForQueue(_defaultAlarms, resource, service, groupParameters, queueResource);
return(alarms);
}
public async Task <IList <AlarmDefinition> > CopyAndUpdateDefaultAlarmsForResource(
AwsResource <T> entity,
IList <AlarmDefinition> alerts,
AwsServiceAlarms <TAlarmConfig> serviceConfig,
ResourceThresholds <TAlarmConfig> resourceConfig)
{
var result = new List <AlarmDefinition>();
foreach (var defaultAlarm in alerts)
{
var alarm = await AlarmWithMergedValues(entity, defaultAlarm, serviceConfig, resourceConfig);
result.Add(alarm);
}
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <DynamoResourceConfig> resource,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var result = await BuildTableAlarms(resource, service, groupParameters, entity);
result.AddRange(await BuildIndexAlarms(resource, service, groupParameters, entity));
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var mergedConfig = service.Options.OverrideWith(resource.Options);
var mergedValuesByAlarmName = service.Values.OverrideWith(resource.Values);
var result = new List <Alarm>();
foreach (var alarm in _defaultAlarms)
{
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
entity.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <DynamoResourceConfig> resource,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
_logger.Error($"Skipping table {resource.Name} as it does not exist");
return(Array.Empty <Alarm>());
}
var result = await BuildTableAlarms(resource, service, groupParameters, entity);
result.AddRange(await BuildIndexAlarms(resource, service, groupParameters, entity));
return(result);
}
private static void ValidateResource(string agName, string serviceName, ResourceThresholds resource)
{
if (resource == null)
{
throw new ConfigException($"AlertingGroup '{agName}' has a '{serviceName}' Service with null resource");
}
if (string.IsNullOrWhiteSpace(resource.Name) && string.IsNullOrWhiteSpace(resource.Pattern))
{
throw new ConfigException(
$"AlertingGroup '{agName}' has a '{serviceName}' Service with no name or pattern");
}
if (resource.Values != null)
{
foreach (var threshold in resource.Values)
{
ValidServiceThreshold(threshold);
}
}
}
private async Task <IList <Alarm> > BuildIndexAlarms(ResourceThresholds <ResourceConfig> resource, AwsServiceAlarms <ResourceConfig> service, AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var result = new List <Alarm>();
var gsiSet = entity.Resource.GlobalSecondaryIndexes;
foreach (var gsi in gsiSet)
{
var gsiResource = new AwsResource <GlobalSecondaryIndexDescription>(gsi.IndexName, gsi);
var expandedGsi = await _gsiAlarmBuilder.CopyAndUpdateDefaultAlarmsForResource(
gsiResource,
Defaults.DynamoDbGsi, service, resource);
foreach (var gsiAlarm in expandedGsi)
{
var dimensions = _gsiDimensionProvider.GetDimensions(gsi, gsiAlarm.DimensionNames);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{gsi.IndexName}-{gsiAlarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = _builder.GetAlarmDescription(groupParameters),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = gsiAlarm
};
result.Add(model);
}
}
return(result);
}
private async Task <IList <ResourceThresholds> > GetPatternMatches(ResourceThresholds resourcePattern, string alertingGroupName)
{
var tableNames = await _resourceSource.GetResourceNamesAsync();
var matches = tableNames
.WhereRegexIsMatch(resourcePattern.Pattern)
.Select(tn => PatternToTable(resourcePattern, tn))
.ToList();
if (matches.Count == 0)
{
_logger.Info($"{alertingGroupName} pattern '{resourcePattern.Pattern}' matched no resource names");
}
else if (matches.Count == tableNames.Count)
{
_logger.Info($"{alertingGroupName} pattern '{resourcePattern.Pattern}' matched all resource names");
}
else
{
_logger.Info($"{alertingGroupName} pattern '{resourcePattern.Pattern}' matched {matches.Count} of {tableNames.Count} resource names");
}
return(matches);
}
public ResourceAndThresholdsPair(ResourceThresholds <TConfig> config, AwsResource <TResource> resource)
{
Definition = config;
Resource = resource;
}
private async Task <List <Alarm> > BuildTableAlarms(IList <AlarmDefinition> defaults, ResourceThresholds <ResourceConfig> resource, AwsServiceAlarms <ResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var expanded = await _builder.CopyAndUpdateDefaultAlarmsForResource(entity, defaults, service, resource);
var result = new List <Alarm>();
foreach (var alarm in expanded)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = _builder.GetAlarmDescription(groupParameters),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = alarm
};
result.Add(model);
}
return(result);
}
public async Task PopulateResourceNames_ByPatternAndNamed_ResourcesExpandedWithCorrectEvaluationPeriods()
{
// arrange
var resourceSourceStub = new Mock <IResourceSource <ExampleServiceModel> >();
resourceSourceStub
.Setup(x => x.GetResourceNamesAsync())
.ReturnsAsync(new List <string>
{
"ItemX",
"ItemY",
"ItemZ"
});
var sut = new ResourceNamePopulator <ExampleServiceModel, ResourceConfig>(new ConsoleAlarmLogger(false), resourceSourceStub.Object);
var namedItem = new ResourceThresholds <ResourceConfig>
{
Name = "ItemX",
Values = new Dictionary <string, AlarmValues>
{
{
"SomeThreshold", new AlarmValues(500, 2)
}
}
};
var patternMatchedItem = new ResourceThresholds <ResourceConfig>
{
Pattern = "Item",
Values = new Dictionary <string, AlarmValues>
{
{
"SomeThreshold", new AlarmValues(100, 3)
}
}
};
var group = new ServiceAlertingGroup <ResourceConfig>
{
GroupParameters = new AlertingGroupParameters("name", "suffix"),
Service = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >
{
namedItem,
patternMatchedItem
}
}
};
// act
await sut.PopulateResourceNames(group);
// assert
var resources = group.Service.Resources;
Assert.That(resources.Count, Is.EqualTo(3));
var itemXThreshold = resources.First(x => x.Name == "ItemX").Values["SomeThreshold"];
var itemYThreshold = resources.First(x => x.Name == "ItemY").Values["SomeThreshold"];
var itemZThreshold = resources.First(x => x.Name == "ItemZ").Values["SomeThreshold"];
Assert.That(itemXThreshold.EvaluationPeriods, Is.EqualTo(2));
Assert.That(itemYThreshold.EvaluationPeriods, Is.EqualTo(3));
Assert.That(itemZThreshold.EvaluationPeriods, Is.EqualTo(3));
}
