public async Task DoesDeployMultipleStacksIfSelected()
{
// first create a stack which has a resource
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1"
}
}
}
},
2);
var cloudformation = new FakeCloudFormation();
var firstTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
await firstTestContext.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// check it got deployed
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
var stack2 = cloudformation
.Stack("Watchman-test-1");
Assert.That(stack2, Is.Not.Null);
Assert.That(stack2.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
}
public async Task CanUseExtendedStatisticsForResource()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
{
"LatencyHigh", new AlarmValues(extendedStatistic: "p97")
}
}
}
}
}
}
);
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByElb = fakeCloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("LoadBalancerName");
var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));
Assert.That(alarm, Is.Not.Null);
Assert.That(alarm.Properties.ContainsKey("Statistic"), Is.False);
Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97"));
}
public async Task AlarmWithIncludeErrorQueuesFalseWillNotCreateErrorQueueAlarms()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue",
Options = new SqsResourceConfig()
{
IncludeErrorQueues = false
},
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue",
"first-sqs-queue_error"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
Assert.That(alarmsByQueue.ContainsKey("first-sqs-queue"), Is.True);
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(
alarmsForQueue.Exists(
alarm => alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible")
);
Assert.That(!alarmsByQueue.ContainsKey("first-sqs-queue_error"));
}
public async Task <AlbTestSetupData> Build()
{
if (!_loadBalancers.Any())
{
WithDefaultLoadBalancer();
}
var config = ConfigHelper.CreateBasicConfiguration(_configurationName, _configurationSuffix,
new AlertingGroupServices()
{
Alb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = _pattern
}
},
Values = _overrides
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancingV2>()
.Setup(x => x.DescribeLoadBalancersAsync(It.IsAny <DescribeLoadBalancersRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(() => new DescribeLoadBalancersResponse
{
LoadBalancers = _loadBalancers
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
var alarms = cloudFormation
.Stack($"Watchman-{_configurationName}")
?.Resources
?.Where(x => x.Value.Type.Equals("AWS::CloudWatch::Alarm", StringComparison.InvariantCultureIgnoreCase))
.Select(x => x.Value)
.ToList();
return(new AlbTestSetupData
{
ConfigurationSuffix = _configurationSuffix,
FakeCloudFormation = cloudFormation,
LoadBalancers = _loadBalancers,
Alarms = alarms
});
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
CloudFront = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >
{
new ResourceThresholds <ResourceConfig>
{
Name = "distribution-1",
Values = new Dictionary <string, AlarmValues>
{
{ "4xxErrorRate", new AlarmValues(10, TimeSpan.FromMinutes(5).Minutes) }
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudFront>().HasCloudFrontDistributions(new[]
{
new DistributionSummary
{
Id = "distribution-1"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
var alarmsByDistributionId = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("DistributionId");
Assert.That(alarmsByDistributionId.ContainsKey("distribution-1"), Is.True);
var alarms = alarmsByDistributionId["distribution-1"];
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "4xxErrorRate" &&
alarm.Properties["AlarmName"].Value <string>().Contains("4xxErrorRate") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix")
));
}
public async Task CanOverrideThreshold()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1",
Values = new Dictionary <string, AlarmValues>()
{
{ "GroupInServiceInstancesLow", 10 }
}
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("AutoScalingGroupName");
var alarm = alarms["group-1"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
Assert.That((decimal)alarm.Properties["Threshold"], Is.EqualTo(40 * 0.1));
}
public async Task GuessesErrorQueueNameWhenNotReportedByCloudWatch()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue"
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(alarmsForQueue, Is.Not.Empty);
var alarmsForErrorQueue = alarmsByQueue["first-sqs-queue_error"];
Assert.That(alarmsForErrorQueue, Is.Not.Empty);
}
public async Task DoesNotDeployNewEmptyStack()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "non-existent-resource"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);
var now = DateTime.Parse("2018-01-26");
ioc.GetMock <ICurrentTimeProvider>()
.Setup(f => f.UtcNow)
.Returns(now);
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Null);
}
public async Task CanOverrideThresholdPercentage()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = "first-table",
Values = new Dictionary <string, AlarmValues>()
{
{ "GsiConsumedReadCapacityUnitsHigh", 20 },
{ "ConsumedReadCapacityUnitsHigh", 10 }
}
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "production-first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByGsi = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("GlobalSecondaryIndexName");
var gsiAlarms = alarmsByGsi["first-gsi"];
var consumedReadGsi = gsiAlarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiConsumedReadCapacityUnitsHigh"));
Assert.That(consumedReadGsi.Properties["Threshold"].Value <int>(),
Is.EqualTo(400 * OneMinuteInSeconds * 0.2m));
var alarmsByTable = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
var tableAlarms = alarmsByTable["production-first-table"];
var consumedReadForTable = tableAlarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-ConsumedReadCapacityUnitsHigh"));
Assert.That(consumedReadForTable.Properties["Threshold"].Value <int>(),
Is.EqualTo(100 * OneMinuteInSeconds * 0.1m));
}
public async Task AlarmsCanBeAddedToGlobalSecondaryIndexes()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-table"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
const decimal defaultCapacityThreshold = 0.8m;
const decimal capacityMultiplier = defaultCapacityThreshold * OneMinuteInSeconds;
const int defaultThrottleThreshold = 2;
// basic check we still have table alarms
Assert.That(cloudformation.Stack("Watchman-test").AlarmsByDimension("TableName").Any(), Is.True);
var alarmsByGsi = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("GlobalSecondaryIndexName");
Assert.That(alarmsByGsi.ContainsKey("first-gsi"), Is.True);
var alarms = alarmsByGsi["first-gsi"];
var consumedRead = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiConsumedReadCapacityUnitsHigh"));
Assert.That(consumedRead, Is.Not.Null, "GSI read alarm missing");
Assert.That(consumedRead.Properties["MetricName"].Value <string>(), Is.EqualTo("ConsumedReadCapacityUnits"));
Assert.That(consumedRead.Properties["Threshold"].Value <int>(),
Is.EqualTo(400 * capacityMultiplier));
Assert.That(consumedRead.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(consumedRead.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(consumedRead.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(consumedRead.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(consumedRead.Dimension("TableName"), Is.EqualTo("first-table"));
var consumedWrite = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiConsumedWriteCapacityUnitsHigh"));
Assert.That(consumedWrite, Is.Not.Null, "GSI write alarm missing");
Assert.That(consumedWrite.Properties["MetricName"].Value <string>(),
Is.EqualTo("ConsumedWriteCapacityUnits"));
Assert.That(consumedWrite.Properties["Threshold"].Value <int>(),
Is.EqualTo(500 * capacityMultiplier));
Assert.That(consumedWrite.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(consumedWrite.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(consumedWrite.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(consumedWrite.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(consumedWrite.Dimension("TableName"), Is.EqualTo("first-table"));
var readThrottle = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiReadThrottleEventsHigh"));
Assert.That(readThrottle, Is.Not.Null, "GSI read throttle alarm missing");
Assert.That(readThrottle.Properties["MetricName"].Value <string>(), Is.EqualTo("ReadThrottleEvents"));
Assert.That(readThrottle.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(readThrottle.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(readThrottle.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(readThrottle.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(readThrottle.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(readThrottle.Dimension("TableName"), Is.EqualTo("first-table"));
var writeThrottle = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiWriteThrottleEventsHigh"));
Assert.That(writeThrottle, Is.Not.Null, "GSI write throttle alarm missing");
Assert.That(writeThrottle.Properties["MetricName"].Value <string>(), Is.EqualTo("WriteThrottleEvents"));
Assert.That(writeThrottle.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(writeThrottle.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(writeThrottle.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(writeThrottle.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(writeThrottle.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(writeThrottle.Dimension("TableName"), Is.EqualTo("first-table"));
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-dynamo-table"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
const decimal defaultCapacityThreshold = 0.8m;
const decimal capacityMultiplier = defaultCapacityThreshold * OneMinuteInSeconds;
const int defaultThrottleThreshold = 2;
var alarmsByTable = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarms = alarmsByTable["first-dynamo-table"];
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ConsumedReadCapacityUnits" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ConsumedReadCapacityUnitsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 100 * capacityMultiplier &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.DynamoDb
)
);
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ConsumedWriteCapacityUnits" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ConsumedWriteCapacityUnitsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 200 * capacityMultiplier &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.DynamoDb
)
);
var readThrottleAlarm =
alarms.Single(a => a.Properties["MetricName"].Value <string>() == "ReadThrottleEvents");
Assert.That(readThrottleAlarm.Properties["AlarmName"].Value <string>(),
Contains.Substring("ReadThrottleEventsHigh"));
Assert.That(readThrottleAlarm.Properties["AlarmName"].Value <string>(), Contains.Substring("-group-suffix"));
Assert.That(readThrottleAlarm.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(readThrottleAlarm.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(readThrottleAlarm.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(readThrottleAlarm.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(readThrottleAlarm.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
var writeThrottleAlarm =
alarms.Single(a => a.Properties["MetricName"].Value <string>() == "WriteThrottleEvents");
Assert.That(writeThrottleAlarm.Properties["AlarmName"].Value <string>(),
Contains.Substring("WriteThrottleEventsHigh"));
Assert.That(writeThrottleAlarm.Properties["AlarmName"].Value <string>(),
Contains.Substring("-group-suffix"));
Assert.That(writeThrottleAlarm.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(writeThrottleAlarm.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(writeThrottleAlarm.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(writeThrottleAlarm.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(writeThrottleAlarm.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
}
public async Task AlarmsHaveMeaningfulDescription()
{
// arrange
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "group-with-description",
AlarmNameSuffix = "group-suffix-1",
Description = "Group description for group 1",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1"
}
}
}
}
},
new AlertingGroup()
{
Name = "group-without-description",
AlarmNameSuffix = "group-suffix-2",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-2"
}
}
}
}
}
}
};
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var firstGroupAlarm = fakeCloudFormation
.Stack("Watchman-group-with-description")
.Alarms()
.First();
var description = firstGroupAlarm.Properties["AlarmDescription"].ToString();
Assert.That(description, Contains.Substring("Alarm generated by AwsWatchman"));
Assert.That(description, Contains.Substring("group-with-description"));
Assert.That(description, Contains.Substring("Group description for group 1"));
var secondGroupAlarm = fakeCloudFormation
.Stack("Watchman-group-without-description")
.Alarms()
.First();
var description2 = secondGroupAlarm.Properties["AlarmDescription"].ToString();
Assert.That(description2, Contains.Substring("Alarm generated by AwsWatchman"));
Assert.That(description2, Contains.Substring("group-without-description"));
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Dax = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >
{
new ResourceThresholds <ResourceConfig>
{
Name = "first-dax-cluster",
Values = new Dictionary <string, AlarmValues>
{
{ "CPUUtilizationHigh", new AlarmValues(10) }
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDAX>().HasClusters(new[]
{
new Cluster
{
ClusterName = "first-dax-cluster",
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByCluster = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("ClusterId");
Assert.That(alarmsByCluster.ContainsKey("first-dax-cluster"), Is.True);
var alarms = alarmsByCluster["first-dax-cluster"];
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "CPUUtilization" &&
alarm.Properties["AlarmName"].Value <string>().Contains("CPUUtilizationHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 10 &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Maximum" &&
alarm.Properties["Namespace"].Value <string>() == "AWS/DAX" &&
alarm.Properties["TreatMissingData"].Value <string>() == "missing"
)
);
}
public async Task AlarmCreatedWithCorrectDefaults()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices
{
Lambda = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = "lambda-test"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration {
FunctionName = "lambda-test", Timeout = 10
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
var alarmsByFunction = stack
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("lambda-test"), Is.True);
var alarmsForLambda = alarmsByFunction["lambda-test"];
Assert.That(alarmsForLambda.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "Errors" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ErrorsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 3 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 1 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Lambda &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.NotBreaching
)
);
Assert.That(alarmsForLambda.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "Duration" &&
alarm.Properties["AlarmName"].Value <string>().Contains("DurationHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 10 * 1000 * 50 / 100 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 1 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Average" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Lambda &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
}
public async Task IgnoresTablesThatOnlyExistInListings()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = "^.*$"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
var dynamo = ioc.GetMock <IAmazonDynamoDB>();
dynamo
.Setup(x => x.ListTablesAsync(It.IsAny <string>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new ListTablesResponse()
{
TableNames = new List <string>()
{
"non-existent", "existent"
}
});
dynamo
.Setup(x => x.DescribeTableAsync("non-existent", It.IsAny <CancellationToken>()))
.ThrowsAsync(new ResourceNotFoundException("bad"))
// so we know this actually got hit and the error was thrown
.Verifiable();
dynamo
.Setup(x => x.DescribeTableAsync("existent", It.IsAny <CancellationToken>()))
.ReturnsAsync(new DescribeTableResponse()
{
Table = new TableDescription()
{
TableName = "existent",
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>(),
ProvisionedThroughput = new ProvisionedThroughputDescription()
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
dynamo.Verify();
Assert.That(cloudformation.StacksDeployed, Is.EqualTo(1));
var alarms = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarms.Keys.ToArray(), Is.EquivalentTo(new[] { "existent" }));
}
public async Task OptionsConfigInParentPatternAppliedWhenNoChild()
{
var consumedReadCapacityUnitsHigh = 900;
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = "first-table",
Values = new Dictionary <string, AlarmValues>()
{
{ "ConsumedReadCapacityUnitsHigh", consumedReadCapacityUnitsHigh }
}
}
},
Options = new DynamoResourceConfig()
{
ThresholdIsAbsolute = true,
MonitorWrites = false
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "production-first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 1000,
WriteCapacityUnits = 2000
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByTable = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
var tableAlarms = alarmsByTable["production-first-table"];
var consumedReadForTable = tableAlarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-ConsumedReadCapacityUnitsHigh"));
Assert.That(consumedReadForTable.Properties["Threshold"].Value <int>(),
Is.EqualTo(consumedReadCapacityUnitsHigh));
var consumedWriteForTable = tableAlarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-ConsumedWriteCapacityUnitsHigh"));
Assert.That(consumedWriteForTable, Is.Null);
}
public async Task DoesDeployEmptyStackIfAlreadyPresentButResourcesNoLongerExist()
{
// first create a stack which has a resource
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var firstTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
await firstTestContext.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// check it got deployed
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
var secondTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
// no matching resource so the next run results in an empty stack
secondTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);
await secondTestContext
.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
stack = cloudformation
.Stack("Watchman-test");
//check we deployed an empty stack and deleted the redundant resources
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources.Any(x => x.Value.Type == "AWS::CloudWatch::Alarm"), Is.False);
}
public async Task AlarmWithManuallySetThresholdRetainsThatValueForTheRelevantAlarm()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue",
Values = new Dictionary <string, AlarmValues>()
{
{ "NumberOfVisibleMessages_Error", new AlarmValues(value: 1) }
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue",
"first-sqs-queue_error"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
Assert.That(alarmsByQueue.ContainsKey("first-sqs-queue"), Is.True);
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["Threshold"].Value <int>() == 100
)
);
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["Threshold"].Value <int>() == 600
)
);
var alarmsForErrorQueue = alarmsByQueue["first-sqs-queue_error"];
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["Threshold"].Value <int>() == 1
)
);
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["Threshold"].Value <int>() == 600
)
);
}
public async Task UsesDesiredInstancesForThresholdIfCloudWatchMetricsDoNotExist()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1",
Options = new AutoScalingResourceConfig()
{
InstanceCountIncreaseDelayMinutes = 10
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
var now = DateTime.UtcNow;
ioc.GetMock <ICurrentTimeProvider>().Setup(a => a.UtcNow).Returns(now);
ioc.GetMock <IAmazonCloudWatch>().Setup(x =>
x.GetMetricStatisticsAsync(It.IsAny <GetMetricStatisticsRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new GetMetricStatisticsResponse()
{
Datapoints = new List <Datapoint>()
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("AutoScalingGroupName");
var alarm = alarms["group-1"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
var defaultAlarmThreshold = 0.5m;
Assert.That((decimal)alarm.Properties["Threshold"], Is.EqualTo(40 * defaultAlarmThreshold));
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-dynamo-table"
}
}
},
Lambda = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-lambda-function"
}
}
}
}
);
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration()
{
FunctionName = "first-lambda-function"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByTable = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarmsByFunction = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True);
}
public async Task AlarmCreatedWithCorrectOverrides()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices
{
Lambda = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = "lambda-test",
Values = new Dictionary <string, AlarmValues>
{
{
"ErrorsHigh", new AlarmValues
(
evaluationPeriods: 3,
periodMinutes: 10,
value: 100
)
}
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration {
FunctionName = "lambda-test", Timeout = 10
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
var alarmsByFunction = stack
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("lambda-test"), Is.True);
var alarmsForLambda = alarmsByFunction["lambda-test"];
Assert.That(alarmsForLambda.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "Errors" &&
alarm.Properties["Period"].Value <int>() == 60 * 10 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 3 &&
alarm.Properties["Threshold"].Value <int>() == 100)
);
}
public async Task GsiLogicalResourceNameContainsTable()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = "first-table",
Values = new Dictionary <string, AlarmValues>()
{
{ "GsiConsumedReadCapacityUnitsHigh", 20 },
{ "ConsumedReadCapacityUnitsHigh", 10 }
}
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "production-first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var resources = cloudFormation
.Stack("Watchman-test")
.Resources;
// TODO: logical name should include table and GSI name
Assert.That(resources, Contains.Key("productionfirsttableGsiConsumedReadCapacityUnitsHigh"));
}
public async Task CanDisableWritesAcrossTableAndGsi()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-table"
}
},
Options = new DynamoResourceConfig()
{
MonitorWrites = false
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = cloudformation
.Stack("Watchman-test")
.Alarms();
// check we didn't disable read alarms
Assert.That(alarms.Count(
a => a.Properties["MetricName"].ToString()
.Contains("ReadThrottleEvents")), Is.AtLeast(2));
Assert.That(alarms.Count(
a => a.Properties["MetricName"].ToString()
.Contains("ConsumedReadCapacityUnits")), Is.AtLeast(2));
// no write alarms
Assert.That(alarms.Where(
a => a.Properties["MetricName"].ToString()
== "ConsumedWriteCapacityUnits"), Is.Empty);
Assert.That(alarms.Where(
a => a.Properties["MetricName"].ToString()
== "WriteThrottleEvents"), Is.Empty);
}
public async Task AlarmCreatedWithCorrectOverrides()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices
{
Lambda = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = "rdscluster-test",
Values = new Dictionary <string, AlarmValues>
{
{
"CPUUtilizationHigh", new AlarmValues
(
evaluationPeriods: 1,
periodMinutes: 5,
value: 70
)
}
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonRDS>().HasRdsClusters(new[]
{
new DBCluster {
DBClusterIdentifier = "rdscluster-test"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
var alarmsByRdscluster = stack
.AlarmsByDimension("DBClusterIdentifier");
Assert.That(alarmsByRdscluster.ContainsKey("rdscluster-test"), Is.True);
var alarmsForCluster = alarmsByRdscluster["rdscluster-test"];
Assert.That(alarmsForCluster.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "CPUUtilization" &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 1 &&
alarm.Properties["Threshold"].Value <int>() == 70)
);
}
public async Task AlarmCreatedWithCorrectDefaults()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices
{
RdsCluster = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = "rdscluster-test"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonRDS>().HasRdsClusters(new[]
{
new DBCluster {
DBClusterIdentifier = "rdscluster-test"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
var alarmsByRdscluster = stack
.AlarmsByDimension("DBClusterIdentifier");
Assert.That(alarmsByRdscluster.ContainsKey("rdscluster-test"), Is.True);
var alarmsForCluster = alarmsByRdscluster["rdscluster-test"];
Assert.That(alarmsForCluster.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "CPUUtilization" &&
alarm.Properties["AlarmName"].Value <string>().Contains("CPUUtilizationHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 60 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Maximum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Rds &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
}
public async Task UsesDelayedScalingThresholdFromCloudWatch()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-delay-20",
Options = new AutoScalingResourceConfig()
{
InstanceCountIncreaseDelayMinutes = 20
}
},
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-delay-100"
}
},
Options = new AutoScalingResourceConfig()
{
InstanceCountIncreaseDelayMinutes = 100
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-delay-20"
},
new AutoScalingGroup()
{
AutoScalingGroupName = "group-delay-100"
}
});
var now = DateTime.Parse("2018-01-26");
ioc.GetMock <ICurrentTimeProvider>()
.Setup(f => f.UtcNow)
.Returns(now);
var cloudWatch = ioc.GetMock <IAmazonCloudWatch>();
SetupCloudWatchDesiredMetric(cloudWatch, 100 * 60, now, "group-delay-100", 90);
SetupCloudWatchDesiredMetric(cloudWatch, 20 * 60, now, "group-delay-20", 80);
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("AutoScalingGroupName");
var alarm100 = alarms["group-delay-100"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
var alarm20 = alarms["group-delay-20"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
var defaultAlarmThreshold = 0.5m;
Assert.That((decimal)alarm100.Properties["Threshold"], Is.EqualTo(90 * defaultAlarmThreshold));
Assert.That((decimal)alarm20.Properties["Threshold"], Is.EqualTo(80 * defaultAlarmThreshold));
}
public async Task CanOptOut()
{
// arrange
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "group-with-description",
AlarmNameSuffix = "group-suffix-1",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
// to test we can opt out at resource level
{ "LatencyHigh", new AlarmValues(enabled: false) },
// to test we can revert an opt-out at service level
{ "UnHealthyHostCountHigh", new AlarmValues(enabled: true) }
}
}
},
Values = new Dictionary <string, AlarmValues>()
{
// test we can opt out at service level
{ "Http5xxErrorsHigh", new AlarmValues(enabled: false) },
// setup for above (we can revert an opt-out at service level)
{ "UnHealthyHostCountHigh", new AlarmValues(enabled: false) }
}
}
}
}
}
};
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = fakeCloudFormation
.Stack("Watchman-group-with-description")
.AlarmsByDimension("LoadBalancerName");
var alarmsForElb1 = alarms["elb-1"];
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("LatencyHigh")
), Is.False);
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("Http5xxErrorsHigh")
), Is.False);
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("UnHealthyHostCountHigh")
), Is.True);
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue",
"first-sqs-queue_error"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
Assert.That(alarmsByQueue.ContainsKey("first-sqs-queue"), Is.True);
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["AlarmName"].Value <string>().Contains("NumberOfVisibleMessages") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 100 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Average" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["AlarmName"].Value <string>().Contains("AgeOfOldestMessage") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 600 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Maximum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
var alarmsForErrorQueue = alarmsByQueue["first-sqs-queue_error"];
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["AlarmName"].Value <string>().Contains("NumberOfVisibleMessages_Error") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 10 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Average" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["AlarmName"].Value <string>().Contains("AgeOfOldestMessage_Error") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 600 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Maximum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
}