private async Task <IList <Alarm> > CreateAlarmsForResource(
IList <AlarmDefinition> defaults,
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
// apply thresholds from resource or alerting group
var expanded = await _builder.CopyAndUpdateDefaultAlarmsForResource(entity, defaults, service, resource);
var result = new List <Alarm>();
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
foreach (var alarm in expanded)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = _builder.GetAlarmDescription(groupParameters),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = alarm
};
result.Add(model);
}
return(result);
}
private async Task <IList <Alarm> > BuildIndexAlarms(ResourceAndThresholdsPair <DynamoResourceConfig, TableDescription> resourceConfig,
PopulatedServiceAlarms <DynamoResourceConfig, TableDescription> service,
AlertingGroupParameters groupParameters,
TableDescription parentTableDescription)
{
// called twice
var mergedConfig = service.Options.OverrideWith(resourceConfig.Definition.Options);
var result = new List <Alarm>();
var gsiSet = parentTableDescription.GlobalSecondaryIndexes;
var mergedValuesByAlarmName = service.Values
.OverrideWith(resourceConfig.Definition.Values);
var defaults = _defaultAlarms.DynamoDbGsiRead;
if (mergedConfig.MonitorWrites ?? DynamoResourceConfig.MonitorWritesDefault)
{
defaults = defaults.Concat(_defaultAlarms.DynamoDbGsiWrite).ToArray();
}
foreach (var gsi in gsiSet)
{
foreach (var alarm in defaults)
{
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var dimensions = _gsiProvider.GetDimensions(gsi, parentTableDescription, alarm.DimensionNames);
if (mergedConfig.ThresholdIsAbsolute ?? DynamoResourceConfig.ThresholdIsAbsoluteDefault)
{
configuredThreshold.ThresholdType = ThresholdType.Absolute;
}
var threshold = await ThresholdCalculator.ExpandThreshold(_gsiProvider,
gsi,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resourceConfig.Resource.Name}-{gsi.IndexName}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(resourceConfig.Definition),
ResourceIdentifier = resourceConfig.Resource.Name,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
}
return(result);
}
public static string DefaultAlarmDescription(this AlertingGroupParameters groupParameters)
{
var suffix = string.IsNullOrWhiteSpace(groupParameters.Description)
? null
: $" ({groupParameters.Description})";
var description = $"{AwsConstants.DefaultDescription}. Alerting group: {groupParameters.Name}{suffix}";
return(description);
}
private async Task <IList <Alarm> > BuildIndexAlarms(ResourceThresholds <DynamoResourceConfig> resourceConfig,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> parentTableEntity)
{
// called twice
var mergedConfig = service.Options.OverrideWith(resourceConfig.Options);
var result = new List <Alarm>();
var gsiSet = parentTableEntity.Resource.GlobalSecondaryIndexes;
var mergedValuesByAlarmName = service.Values.OverrideWith(resourceConfig.Values);
var defaults = _defaultAlarms.DynamoDbGsiRead;
if (mergedConfig.MonitorWrites ?? DynamoResourceConfig.MonitorWritesDefault)
{
defaults = defaults.Concat(_defaultAlarms.DynamoDbGsiWrite).ToArray();
}
foreach (var gsi in gsiSet)
{
var gsiResource = new AwsResource <GlobalSecondaryIndexDescription>(gsi.IndexName, gsi);
foreach (var alarm in defaults)
{
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var dimensions = _gsiProvider.GetDimensions(gsi, parentTableEntity.Resource, alarm.DimensionNames);
var threshold = await ThresholdCalculator.ExpandThreshold(_gsiProvider,
gsiResource.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resourceConfig.Name}-{gsi.IndexName}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
// TODO: remove this property in a future PR
// passing in the entity shouldn't be necessary and passing in the table entity here
// when the alarm is for a GSI is an even worse hack
Resource = parentTableEntity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
}
return(result);
}
private async Task <List <Alarm> > BuildAlarmsForQueue(
IList <AlarmDefinition> defaults,
ResourceThresholds <SqsResourceConfig> resource,
PopulatedServiceAlarms <SqsResourceConfig, QueueDataV2> service,
AlertingGroupParameters groupParameters,
QueueDataV2 queue)
{
var mergedConfig = service.Options.OverrideWith(resource.Options);
bool includeErrorQueues = mergedConfig.IncludeErrorQueues ?? true;
var result = new List <Alarm>();
var mergedValuesByAlarmName = service.Values.OverrideWith(resource.Values);
foreach (var alarm in defaults)
{
var dimensions = _dimensionProvider.GetDimensions(queue, alarm.DimensionNames);
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var actualThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
queue,
mergedConfig,
actualThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(resource),
// error queues currently named as per parent queue
ResourceIdentifier = resource.Name,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
if (includeErrorQueues && queue.ErrorQueue != null)
{
var alarms = await BuildAlarmsForQueue(_errorQueueDefaults, resource, service, groupParameters,
queue.ErrorQueue);
result.AddRange(alarms);
}
return(result);
}
private async Task <List <Alarm> > BuildTableAlarms(ResourceThresholds <DynamoResourceConfig> resourceConfig,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var mergedConfig = service.Options.OverrideWith(resourceConfig.Options);
var result = new List <Alarm>();
var mergedValuesByAlarmName = service.Values.OverrideWith(resourceConfig.Values);
var defaults = _defaultAlarms.DynamoDbRead;
if (mergedConfig.MonitorWrites ?? DynamoResourceConfig.MonitorWritesDefault)
{
defaults = defaults.Concat(_defaultAlarms.DynamoDbWrite).ToArray();
}
foreach (var alarm in defaults)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
if (mergedConfig.ThresholdIsAbsolute ?? DynamoResourceConfig.ThresholdIsAbsoluteDefault)
{
configuredThreshold.ThresholdType = ThresholdType.Absolute;
}
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
entity.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resourceConfig.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
return(result);
}
public static string DefaultAlarmDescription(this AlertingGroupParameters groupParameters, IResource resource)
{
var suffix = string.IsNullOrWhiteSpace(groupParameters.Description)
? null
: $" ({groupParameters.Description})";
var description = $"{AwsConstants.V2DefaultDescription}. Alerting group: {groupParameters.Name}{suffix}";
if (!string.IsNullOrWhiteSpace(resource.Description))
{
description = $"{resource.Description}{Environment.NewLine}{Environment.NewLine}{description}";
}
return(description);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <SqsResourceConfig> resource,
AwsServiceAlarms <SqsResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _queueSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var queueResource = new AwsResource <QueueDataV2>(entity.Name, entity.Resource);
var alarms = await BuildAlarmsForQueue(_defaultAlarms, resource, service, groupParameters, queueResource);
return(alarms);
}
private async Task<IList<Alarm>> CreateAlarmsForResource(
ResourceAndThresholdsPair<SqsResourceConfig, QueueDataV2> resource,
PopulatedServiceAlarms<SqsResourceConfig, QueueDataV2> service,
AlertingGroupParameters groupParameters)
{
var entity = await resource.Resource.GetFullResource();
if (entity == null)
{
throw new Exception($"Entity {resource.Resource.Name} not found");
}
var mergedConfig = service.Options.OverrideWith(resource.Definition.Options);
var includeErrorQueues = mergedConfig.IncludeErrorQueues ?? true;
var result = new List<Alarm>();
// get working queue alarms
if (entity.WorkingQueue != null)
{
var alarms = await BuildAlarmsForQueue(_defaultAlarms,
mergedConfig,
resource.Definition,
service,
groupParameters,
entity.WorkingQueue);
result.AddRange(alarms);
}
// get error queue alarms
if (includeErrorQueues && entity.ErrorQueue != null)
{
var alarms = await BuildAlarmsForQueue(_errorQueueDefaults,
mergedConfig,
resource.Definition,
service,
groupParameters,
entity.ErrorQueue);
result.AddRange(alarms);
}
return result;
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <DynamoResourceConfig> resource,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var result = await BuildTableAlarms(resource, service, groupParameters, entity);
result.AddRange(await BuildIndexAlarms(resource, service, groupParameters, entity));
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <TAlarmConfig> resource,
AwsServiceAlarms <TAlarmConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
throw new Exception($"Entity {resource.Name} not found");
}
var mergedConfig = service.Options.OverrideWith(resource.Options);
var mergedValuesByAlarmName = service.Values.OverrideWith(resource.Values);
var result = new List <Alarm>();
foreach (var alarm in _defaultAlarms)
{
var values = mergedValuesByAlarmName.GetValueOrDefault(alarm.Name) ?? new AlarmValues();
var configuredThreshold = alarm.Threshold.CopyWith(value: values.Threshold);
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var threshold = await ThresholdCalculator.ExpandThreshold(_attributeProvider,
entity.Resource,
mergedConfig,
configuredThreshold);
var built = alarm.CopyWith(threshold, values);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{built.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = groupParameters.DefaultAlarmDescription(),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = built
};
result.Add(model);
}
return(result);
}
public async Task<IList<Alarm>> GenerateAlarmsFor(
PopulatedServiceAlarms<SqsResourceConfig, QueueDataV2> service,
AlertingGroupParameters groupParameters)
{
if (service?.Resources == null || service.Resources.Count == 0)
{
return new List<Alarm>();
}
var alarms = new List<Alarm>();
foreach (var resource in service.Resources)
{
var alarmsForResource = await CreateAlarmsForResource(resource, service, groupParameters);
alarms.AddRange(alarmsForResource);
}
return alarms;
}
public void AddAlarms(AlertingGroupParameters group, IList <Alarm> alarms)
{
foreach (var alarm in alarms)
{
if (alarm.AlarmDefinition.Threshold.ThresholdType != ThresholdType.Absolute)
{
throw new Exception("Threshold type must be absolute for creation");
}
}
if (_alarms.ContainsKey(group))
{
_alarms[group].AddRange(alarms);
}
else
{
_alarms.Add(group, alarms.ToList());
}
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceThresholds <DynamoResourceConfig> resource,
AwsServiceAlarms <DynamoResourceConfig> service,
AlertingGroupParameters groupParameters)
{
var entity = await _tableSource.GetResourceAsync(resource.Name);
if (entity == null)
{
_logger.Error($"Skipping table {resource.Name} as it does not exist");
return(Array.Empty <Alarm>());
}
var result = await BuildTableAlarms(resource, service, groupParameters, entity);
result.AddRange(await BuildIndexAlarms(resource, service, groupParameters, entity));
return(result);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceAndThresholdsPair <DynamoResourceConfig, TableDescription> resource,
PopulatedServiceAlarms <DynamoResourceConfig, TableDescription> service,
AlertingGroupParameters groupParameters)
{
var entity = await resource.Resource.GetFullResource();
if (entity == null)
{
_logger.Error($"Skipping table {resource.Resource.Name} as it does not exist");
return(Array.Empty <Alarm>());
}
var result = await BuildTableAlarms(resource, service, groupParameters, entity);
result.AddRange(await BuildIndexAlarms(resource, service, groupParameters, entity));
return(result);
}
public async Task <IList <Alarm> > GenerateAlarmsFor(
AwsServiceAlarms <TAlarmConfig> service,
AlertingGroupParameters groupParameters)
{
if (service?.Resources == null || service.Resources.Count == 0)
{
return(new List <Alarm>());
}
List <Alarm> alarms = new List <Alarm>();
foreach (var resource in service.Resources)
{
var alarmsForResource = await CreateAlarmsForResource(resource, service, groupParameters);
alarms.AddRange(alarmsForResource);
}
return(alarms);
}
private async Task <IList <Alarm> > CreateAlarmsForResource(
ResourceAndThresholdsPair <SqsResourceConfig, QueueDataV2> resource,
PopulatedServiceAlarms <SqsResourceConfig, QueueDataV2> service,
AlertingGroupParameters groupParameters)
{
var entity = await resource.Resource.GetFullResource();
if (entity == null)
{
throw new Exception($"Entity {resource.Resource.Name} not found");
}
var alarms = await BuildAlarmsForQueue(_defaultAlarms,
resource.Definition,
service,
groupParameters,
entity);
return(alarms);
}
private async Task <List <Alarm> > BuildTableAlarms(IList <AlarmDefinition> defaults, ResourceThresholds <ResourceConfig> resource, AwsServiceAlarms <ResourceConfig> service,
AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var expanded = await _builder.CopyAndUpdateDefaultAlarmsForResource(entity, defaults, service, resource);
var result = new List <Alarm>();
foreach (var alarm in expanded)
{
var dimensions = _dimensions.GetDimensions(entity.Resource, alarm.DimensionNames);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{alarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = _builder.GetAlarmDescription(groupParameters),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = alarm
};
result.Add(model);
}
return(result);
}
private async Task <IList <Alarm> > BuildIndexAlarms(ResourceThresholds <ResourceConfig> resource, AwsServiceAlarms <ResourceConfig> service, AlertingGroupParameters groupParameters,
AwsResource <TableDescription> entity)
{
var result = new List <Alarm>();
var gsiSet = entity.Resource.GlobalSecondaryIndexes;
foreach (var gsi in gsiSet)
{
var gsiResource = new AwsResource <GlobalSecondaryIndexDescription>(gsi.IndexName, gsi);
var expandedGsi = await _gsiAlarmBuilder.CopyAndUpdateDefaultAlarmsForResource(
gsiResource,
Defaults.DynamoDbGsi, service, resource);
foreach (var gsiAlarm in expandedGsi)
{
var dimensions = _gsiDimensionProvider.GetDimensions(gsi, gsiAlarm.DimensionNames);
var model = new Alarm
{
AlarmName = $"{resource.Name}-{gsi.IndexName}-{gsiAlarm.Name}-{groupParameters.AlarmNameSuffix}",
AlarmDescription = _builder.GetAlarmDescription(groupParameters),
Resource = entity,
Dimensions = dimensions,
AlarmDefinition = gsiAlarm
};
result.Add(model);
}
}
return(result);
}
private string ExpectedStackName(AlertingGroupParameters group)
{
return($"Watchman-{group.Name.ToLowerInvariant()}");
}
private string StackName(AlertingGroupParameters group)
{
return("Watchman-" + group.Name.ToLowerInvariant());
}
