public async Task TestNullRefIssueWhenMixingOldAndNewConfig()
{
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "ag1",
AlarmNameSuffix = "ag1suffix",
Services = null,
DynamoDb = new DynamoDb()
{
Tables = new List <Table>()
{
new Table()
{
Name = "banana"
}
}
}
},
new AlertingGroup()
{
Name = "ag2",
AlarmNameSuffix = "ag2suffix",
Services = new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "asg"
}
}
}
}
}
}
};
var cloudformation = new FakeCloudFormation();
var context = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
context.GetMock <IAmazonAutoScaling>()
.HasAutoScalingGroups(new AutoScalingGroup[0]);
await context.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// pass as did not throw
}
public async Task CanUseExtendedStatisticsForResource()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
{
"LatencyHigh", new AlarmValues(extendedStatistic: "p97")
}
}
}
}
}
}
);
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByElb = fakeCloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("LoadBalancerName");
var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));
Assert.That(alarm, Is.Not.Null);
Assert.That(alarm.Properties.ContainsKey("Statistic"), Is.False);
Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97"));
}
public async Task DoesDeployMultipleStacksIfSelected()
{
// first create a stack which has a resource
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1"
}
}
}
},
2);
var cloudformation = new FakeCloudFormation();
var firstTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
await firstTestContext.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// check it got deployed
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
var stack2 = cloudformation
.Stack("Watchman-test-1");
Assert.That(stack2, Is.Not.Null);
Assert.That(stack2.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
}
public async Task AlarmWithIncludeErrorQueuesFalseWillNotCreateErrorQueueAlarms()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue",
Options = new SqsResourceConfig()
{
IncludeErrorQueues = false
},
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue",
"first-sqs-queue_error"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
Assert.That(alarmsByQueue.ContainsKey("first-sqs-queue"), Is.True);
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(
alarmsForQueue.Exists(
alarm => alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible")
);
Assert.That(!alarmsByQueue.ContainsKey("first-sqs-queue_error"));
}
public async Task <AlbTestSetupData> Build()
{
if (!_loadBalancers.Any())
{
WithDefaultLoadBalancer();
}
var config = ConfigHelper.CreateBasicConfiguration(_configurationName, _configurationSuffix,
new AlertingGroupServices()
{
Alb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = _pattern
}
},
Values = _overrides
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancingV2>()
.Setup(x => x.DescribeLoadBalancersAsync(It.IsAny <DescribeLoadBalancersRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(() => new DescribeLoadBalancersResponse
{
LoadBalancers = _loadBalancers
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
var alarms = cloudFormation
.Stack($"Watchman-{_configurationName}")
?.Resources
?.Where(x => x.Value.Type.Equals("AWS::CloudWatch::Alarm", StringComparison.InvariantCultureIgnoreCase))
.Select(x => x.Value)
.ToList();
return(new AlbTestSetupData
{
ConfigurationSuffix = _configurationSuffix,
FakeCloudFormation = cloudFormation,
LoadBalancers = _loadBalancers,
Alarms = alarms
});
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
CloudFront = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >
{
new ResourceThresholds <ResourceConfig>
{
Name = "distribution-1",
Values = new Dictionary <string, AlarmValues>
{
{ "4xxErrorRate", new AlarmValues(10, TimeSpan.FromMinutes(5).Minutes) }
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudFront>().HasCloudFrontDistributions(new[]
{
new DistributionSummary
{
Id = "distribution-1"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
var alarmsByDistributionId = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("DistributionId");
Assert.That(alarmsByDistributionId.ContainsKey("distribution-1"), Is.True);
var alarms = alarmsByDistributionId["distribution-1"];
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "4xxErrorRate" &&
alarm.Properties["AlarmName"].Value <string>().Contains("4xxErrorRate") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix")
));
}
public async Task CanOverrideThreshold()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1",
Values = new Dictionary <string, AlarmValues>()
{
{ "GroupInServiceInstancesLow", 10 }
}
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("AutoScalingGroupName");
var alarm = alarms["group-1"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
Assert.That((decimal)alarm.Properties["Threshold"], Is.EqualTo(40 * 0.1));
}
public async Task GuessesErrorQueueNameWhenNotReportedByCloudWatch()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue"
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(alarmsForQueue, Is.Not.Empty);
var alarmsForErrorQueue = alarmsByQueue["first-sqs-queue_error"];
Assert.That(alarmsForErrorQueue, Is.Not.Empty);
}
public async Task IgnoresNamedEntitiesThatDoNotExist()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "non-existant-table"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 10,
WriteCapacityUnits = 10
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
Assert.That(cloudformation.StacksDeployed, Is.Zero);
}
public async Task DoesNotDeployNewEmptyStack()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "non-existent-resource"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);
var now = DateTime.Parse("2018-01-26");
ioc.GetMock <ICurrentTimeProvider>()
.Setup(f => f.UtcNow)
.Returns(now);
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Null);
}
public async Task StacksAreNotListedMultipleTimes()
{
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Pattern = ".*"
}
}
}
},
numberOfCloudFormationStacks: 5);
var cloudFormation = new FakeCloudFormation();
var context = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = $"group-asg",
DesiredCapacity = 40
}
});
await context.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
var stacks = cloudFormation.Stacks();
Assert.That(stacks.Count, Is.GreaterThan(1));
Assert.That(cloudFormation.CallsToListStacks, Is.EqualTo(1));
}
public async Task IgnoresNamedEntitiesThatDoNotExist()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices
{
Dax = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >
{
new ResourceThresholds <ResourceConfig>
{
Name = "non-existent-cluster"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDAX>().HasClusters(new[]
{
new Cluster
{
ClusterName = "first-dax-cluster"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
Assert.That(cloudformation.StacksDeployed, Is.Zero);
}
public async Task AlarmCreatedWithCorrectDefaults()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices
{
Lambda = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = "lambda-test"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration {
FunctionName = "lambda-test", Timeout = 10
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
var alarmsByFunction = stack
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("lambda-test"), Is.True);
var alarmsForLambda = alarmsByFunction["lambda-test"];
Assert.That(alarmsForLambda.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "Errors" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ErrorsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 3 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 1 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Lambda &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.NotBreaching
)
);
Assert.That(alarmsForLambda.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "Duration" &&
alarm.Properties["AlarmName"].Value <string>().Contains("DurationHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 10 * 1000 * 50 / 100 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 1 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Average" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Lambda &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
}
public async Task AlarmCreatedWithCorrectOverrides()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices
{
Lambda = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = "lambda-test",
Values = new Dictionary <string, AlarmValues>
{
{
"ErrorsHigh", new AlarmValues
(
evaluationPeriods: 3,
periodMinutes: 10,
value: 100
)
}
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration {
FunctionName = "lambda-test", Timeout = 10
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
var alarmsByFunction = stack
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("lambda-test"), Is.True);
var alarmsForLambda = alarmsByFunction["lambda-test"];
Assert.That(alarmsForLambda.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "Errors" &&
alarm.Properties["Period"].Value <int>() == 60 * 10 &&
alarm.Properties["EvaluationPeriods"].Value <int>() == 3 &&
alarm.Properties["Threshold"].Value <int>() == 100)
);
}
public async Task IgnoresTablesThatOnlyExistInListings()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = "^.*$"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
var dynamo = ioc.GetMock <IAmazonDynamoDB>();
dynamo
.Setup(x => x.ListTablesAsync(It.IsAny <string>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new ListTablesResponse()
{
TableNames = new List <string>()
{
"non-existent", "existent"
}
});
dynamo
.Setup(x => x.DescribeTableAsync("non-existent", It.IsAny <CancellationToken>()))
.ThrowsAsync(new ResourceNotFoundException("bad"))
// so we know this actually got hit and the error was thrown
.Verifiable();
dynamo
.Setup(x => x.DescribeTableAsync("existent", It.IsAny <CancellationToken>()))
.ReturnsAsync(new DescribeTableResponse()
{
Table = new TableDescription()
{
TableName = "existent",
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>(),
ProvisionedThroughput = new ProvisionedThroughputDescription()
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
dynamo.Verify();
Assert.That(cloudformation.StacksDeployed, Is.EqualTo(1));
var alarms = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarms.Keys.ToArray(), Is.EquivalentTo(new[] { "existent" }));
}
public async Task CanDisableWritesAcrossTableAndGsi()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-table"
}
},
Options = new DynamoResourceConfig()
{
MonitorWrites = false
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = cloudformation
.Stack("Watchman-test")
.Alarms();
// check we didn't disable read alarms
Assert.That(alarms.Count(
a => a.Properties["MetricName"].ToString()
.Contains("ReadThrottleEvents")), Is.AtLeast(2));
Assert.That(alarms.Count(
a => a.Properties["MetricName"].ToString()
.Contains("ConsumedReadCapacityUnits")), Is.AtLeast(2));
// no write alarms
Assert.That(alarms.Where(
a => a.Properties["MetricName"].ToString()
== "ConsumedWriteCapacityUnits"), Is.Empty);
Assert.That(alarms.Where(
a => a.Properties["MetricName"].ToString()
== "WriteThrottleEvents"), Is.Empty);
}
public async Task GsiLogicalResourceNameContainsTable()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = "first-table",
Values = new Dictionary <string, AlarmValues>()
{
{ "GsiConsumedReadCapacityUnitsHigh", 20 },
{ "ConsumedReadCapacityUnitsHigh", 10 }
}
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "production-first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var resources = cloudFormation
.Stack("Watchman-test")
.Resources;
// TODO: logical name should include table and GSI name
Assert.That(resources, Contains.Key("productionfirsttableGsiConsumedReadCapacityUnitsHigh"));
}
public async Task CanOverrideThresholdPercentage()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = "first-table",
Values = new Dictionary <string, AlarmValues>()
{
{ "GsiConsumedReadCapacityUnitsHigh", 20 },
{ "ConsumedReadCapacityUnitsHigh", 10 }
}
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "production-first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByGsi = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("GlobalSecondaryIndexName");
var gsiAlarms = alarmsByGsi["first-gsi"];
var consumedReadGsi = gsiAlarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiConsumedReadCapacityUnitsHigh"));
Assert.That(consumedReadGsi.Properties["Threshold"].Value <int>(),
Is.EqualTo(400 * OneMinuteInSeconds * 0.2m));
var alarmsByTable = cloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
var tableAlarms = alarmsByTable["production-first-table"];
var consumedReadForTable = tableAlarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-ConsumedReadCapacityUnitsHigh"));
Assert.That(consumedReadForTable.Properties["Threshold"].Value <int>(),
Is.EqualTo(100 * OneMinuteInSeconds * 0.1m));
}
public async Task AlarmWithManuallySetThresholdRetainsThatValueForTheRelevantAlarm()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue",
Values = new Dictionary <string, AlarmValues>()
{
{ "NumberOfVisibleMessages_Error", new AlarmValues(value: 1) }
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue",
"first-sqs-queue_error"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
Assert.That(alarmsByQueue.ContainsKey("first-sqs-queue"), Is.True);
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["Threshold"].Value <int>() == 100
)
);
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["Threshold"].Value <int>() == 600
)
);
var alarmsForErrorQueue = alarmsByQueue["first-sqs-queue_error"];
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["Threshold"].Value <int>() == 1
)
);
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["Threshold"].Value <int>() == 600
)
);
}
public async Task AlarmsCanBeAddedToGlobalSecondaryIndexes()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources =
new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-table"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
},
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "first-gsi",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 400,
WriteCapacityUnits = 500
}
}
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
const decimal defaultCapacityThreshold = 0.8m;
const decimal capacityMultiplier = defaultCapacityThreshold * OneMinuteInSeconds;
const int defaultThrottleThreshold = 2;
// basic check we still have table alarms
Assert.That(cloudformation.Stack("Watchman-test").AlarmsByDimension("TableName").Any(), Is.True);
var alarmsByGsi = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("GlobalSecondaryIndexName");
Assert.That(alarmsByGsi.ContainsKey("first-gsi"), Is.True);
var alarms = alarmsByGsi["first-gsi"];
var consumedRead = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiConsumedReadCapacityUnitsHigh"));
Assert.That(consumedRead, Is.Not.Null, "GSI read alarm missing");
Assert.That(consumedRead.Properties["MetricName"].Value <string>(), Is.EqualTo("ConsumedReadCapacityUnits"));
Assert.That(consumedRead.Properties["Threshold"].Value <int>(),
Is.EqualTo(400 * capacityMultiplier));
Assert.That(consumedRead.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(consumedRead.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(consumedRead.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(consumedRead.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(consumedRead.Dimension("TableName"), Is.EqualTo("first-table"));
var consumedWrite = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiConsumedWriteCapacityUnitsHigh"));
Assert.That(consumedWrite, Is.Not.Null, "GSI write alarm missing");
Assert.That(consumedWrite.Properties["MetricName"].Value <string>(),
Is.EqualTo("ConsumedWriteCapacityUnits"));
Assert.That(consumedWrite.Properties["Threshold"].Value <int>(),
Is.EqualTo(500 * capacityMultiplier));
Assert.That(consumedWrite.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(consumedWrite.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(consumedWrite.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(consumedWrite.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(consumedWrite.Dimension("TableName"), Is.EqualTo("first-table"));
var readThrottle = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiReadThrottleEventsHigh"));
Assert.That(readThrottle, Is.Not.Null, "GSI read throttle alarm missing");
Assert.That(readThrottle.Properties["MetricName"].Value <string>(), Is.EqualTo("ReadThrottleEvents"));
Assert.That(readThrottle.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(readThrottle.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(readThrottle.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(readThrottle.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(readThrottle.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(readThrottle.Dimension("TableName"), Is.EqualTo("first-table"));
var writeThrottle = alarms.SingleOrDefault(
a => a.Properties["AlarmName"].ToString()
.Contains("first-table-first-gsi-GsiWriteThrottleEventsHigh"));
Assert.That(writeThrottle, Is.Not.Null, "GSI write throttle alarm missing");
Assert.That(writeThrottle.Properties["MetricName"].Value <string>(), Is.EqualTo("WriteThrottleEvents"));
Assert.That(writeThrottle.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(writeThrottle.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(writeThrottle.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(writeThrottle.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(writeThrottle.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
Assert.That(writeThrottle.Dimension("TableName"), Is.EqualTo("first-table"));
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-dynamo-table"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
const decimal defaultCapacityThreshold = 0.8m;
const decimal capacityMultiplier = defaultCapacityThreshold * OneMinuteInSeconds;
const int defaultThrottleThreshold = 2;
var alarmsByTable = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarms = alarmsByTable["first-dynamo-table"];
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ConsumedReadCapacityUnits" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ConsumedReadCapacityUnitsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 100 * capacityMultiplier &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.DynamoDb
)
);
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ConsumedWriteCapacityUnits" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ConsumedWriteCapacityUnitsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 200 * capacityMultiplier &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.DynamoDb
)
);
var readThrottleAlarm =
alarms.Single(a => a.Properties["MetricName"].Value <string>() == "ReadThrottleEvents");
Assert.That(readThrottleAlarm.Properties["AlarmName"].Value <string>(),
Contains.Substring("ReadThrottleEventsHigh"));
Assert.That(readThrottleAlarm.Properties["AlarmName"].Value <string>(), Contains.Substring("-group-suffix"));
Assert.That(readThrottleAlarm.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(readThrottleAlarm.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(readThrottleAlarm.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(readThrottleAlarm.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(readThrottleAlarm.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
var writeThrottleAlarm =
alarms.Single(a => a.Properties["MetricName"].Value <string>() == "WriteThrottleEvents");
Assert.That(writeThrottleAlarm.Properties["AlarmName"].Value <string>(),
Contains.Substring("WriteThrottleEventsHigh"));
Assert.That(writeThrottleAlarm.Properties["AlarmName"].Value <string>(),
Contains.Substring("-group-suffix"));
Assert.That(writeThrottleAlarm.Properties["Threshold"].Value <int>(), Is.EqualTo(defaultThrottleThreshold));
Assert.That(writeThrottleAlarm.Properties["Period"].Value <int>(), Is.EqualTo(60));
Assert.That(writeThrottleAlarm.Properties["ComparisonOperator"].Value <string>(),
Is.EqualTo("GreaterThanOrEqualToThreshold"));
Assert.That(writeThrottleAlarm.Properties["Statistic"].Value <string>(), Is.EqualTo("Sum"));
Assert.That(writeThrottleAlarm.Properties["Namespace"].Value <string>(), Is.EqualTo(AwsNamespace.DynamoDb));
}
public async Task AlarmsHaveMeaningfulDescription()
{
// arrange
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "group-with-description",
AlarmNameSuffix = "group-suffix-1",
Description = "Group description for group 1",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1"
}
}
}
}
},
new AlertingGroup()
{
Name = "group-without-description",
AlarmNameSuffix = "group-suffix-2",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-2"
}
}
}
}
}
}
};
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var firstGroupAlarm = fakeCloudFormation
.Stack("Watchman-group-with-description")
.Alarms()
.First();
var description = firstGroupAlarm.Properties["AlarmDescription"].ToString();
Assert.That(description, Contains.Substring("Alarm generated by AwsWatchman"));
Assert.That(description, Contains.Substring("group-with-description"));
Assert.That(description, Contains.Substring("Group description for group 1"));
var secondGroupAlarm = fakeCloudFormation
.Stack("Watchman-group-without-description")
.Alarms()
.First();
var description2 = secondGroupAlarm.Properties["AlarmDescription"].ToString();
Assert.That(description2, Contains.Substring("Alarm generated by AwsWatchman"));
Assert.That(description2, Contains.Substring("group-without-description"));
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-dynamo-table"
}
}
},
Lambda = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-lambda-function"
}
}
}
}
);
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration()
{
FunctionName = "first-lambda-function"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByTable = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarmsByFunction = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True);
}
public async Task DoesDeployEmptyStackIfAlreadyPresentButResourcesNoLongerExist()
{
// first create a stack which has a resource
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var firstTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
await firstTestContext.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// check it got deployed
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
var secondTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
// no matching resource so the next run results in an empty stack
secondTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);
await secondTestContext
.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
stack = cloudformation
.Stack("Watchman-test");
//check we deployed an empty stack and deleted the redundant resources
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources.Any(x => x.Value.Type == "AWS::CloudWatch::Alarm"), Is.False);
}
public async Task OrphanedAlarmsAreReportedCorrectly(bool configuredAlarmsAlreadyExist)
{
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "TestAlertingGroup",
AlarmNameSuffix = "suffix",
Targets = new List <AlertTarget>()
{
new AlertEmail("*****@*****.**")
},
DynamoDb = new DynamoDb()
{
Tables = new List <Table>()
{
new Table()
{
Pattern = "table-with-watchman-alarm"
}
}
},
Sqs = new Configuration.Sqs()
{
Queues = new List <Queue>()
{
new Queue()
{
Pattern = "queue-with-watchman-alarm"
}
},
Errors = new ErrorQueue()
{
Monitored = true
}
}
}
}
});
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new List <TableDescription>()
{
new TableDescription()
{
TableName = "table-with-watchman-alarm",
GlobalSecondaryIndexes = new List <GlobalSecondaryIndexDescription>()
{
new GlobalSecondaryIndexDescription()
{
IndexName = "index",
ProvisionedThroughput = new ProvisionedThroughputDescription()
}
},
ProvisionedThroughput = new ProvisionedThroughputDescription()
}
});
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new List <string>()
{
"queue-with-watchman-alarm",
"queue-with-watchman-alarm_Error"
});
var alarmsPut = new List <string>();
ioc.GetMock <IAmazonCloudWatch>()
.Setup(x => x.PutMetricAlarmAsync(It.IsAny <PutMetricAlarmRequest>(), It.IsAny <CancellationToken>()))
.Callback((PutMetricAlarmRequest a, CancellationToken _) => alarmsPut.Add(a.AlarmName))
.ReturnsAsync(new PutMetricAlarmResponse());
var existingWatchmanAlarmsMatchingResources =
(configuredAlarmsAlreadyExist
? new[]
{
"table-with-watchman-alarm-ConsumedReadCapacityUnits-suffix",
"table-with-watchman-alarm-ReadThrottleEvents-suffix",
"table-with-watchman-alarm-index-ConsumedReadCapacityUnits-suffix",
"table-with-watchman-alarm-index-ReadThrottleEvents-suffix",
"table-with-watchman-alarm-WriteThrottleEvents-suffix",
"table-with-watchman-alarm-index-ConsumedWriteCapacityUnits-suffix",
"table-with-watchman-alarm-index-WriteThrottleEvents-suffix",
"queue-with-watchman-alarm-ApproximateNumberOfMessagesVisible-suffix",
"queue-with-watchman-alarm-ApproximateAgeOfOldestMessage-suffix",
"queue-with-watchman-alarm_Error-ApproximateNumberOfMessagesVisible-suffix",
"queue-with-watchman-alarm_Error-ApproximateAgeOfOldestMessage-suffix"
}
: new string[0])
.Select(x => new MetricAlarm()
{
AlarmDescription = "AwsWatchman",
AlarmName = x
})
.ToArray();
ioc.GetMock <IAmazonCloudWatch>()
.Setup(x => x.DescribeAlarmsAsync(It.IsAny <DescribeAlarmsRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new DescribeAlarmsResponse()
{
MetricAlarms = new List <MetricAlarm>()
{
new MetricAlarm()
{
AlarmDescription = "AwsWatchman",
AlarmName = "orphaned-watchman-alarm"
},
new MetricAlarm()
{
AlarmDescription = "",
AlarmName = "other-service-alarm"
},
new MetricAlarm()
{
AlarmDescription = "AwsWatchman. Alerting Group: test",
AlarmName = "newer-cloudformation-alarm"
}
}.Concat(existingWatchmanAlarmsMatchingResources).ToList()
});
ioc.GetMock <IAmazonSimpleNotificationService>()
.Setup(x => x.CreateTopicAsync(It.IsAny <string>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(new CreateTopicResponse()
{
TopicArn = "topic-arn"
});
var generator = ioc.Get <AlarmLoaderAndGenerator>();
await generator.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
var reporter = ioc.Get <IOrphanedAlarmReporter>();
var reported = await reporter.FindOrphanedAlarms();
Assert.That(alarmsPut, Is.Not.Empty);
Assert.That(reported.Count, Is.EqualTo(1));
Assert.That(reported[0].AlarmName, Is.EqualTo("orphaned-watchman-alarm"));
}
public async Task AlarmsAreDistributedEvenlyAcrossStacks()
{
const int numStacks = 10;
const int numAsgs = 100;
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Pattern = ".*"
}
}
}
},
numberOfCloudFormationStacks: numStacks);
var cloudFormation = new FakeCloudFormation();
var context = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
var lotsOfAsgs = Enumerable.Range(0, numAsgs).Select(r =>
new AutoScalingGroup()
{
AutoScalingGroupName = $"group-{r}",
DesiredCapacity = 40
}
)
.ToArray();
context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(lotsOfAsgs);
try
{
await context.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
}
catch
{
// ignore
}
var stacks = cloudFormation.Stacks();
Assert.That(stacks.Count, Is.EqualTo(numStacks));
var resourceCountsByStack = stacks.Select(s => (s.name, s.template.Resources.Count)).ToArray();
var totalResources = stacks.Sum(s => s.template.Resources.Count);
var alarmCount = stacks
.Sum(s => s.template.Resources.Count(r => r.Value.Type == "AWS::CloudWatch::Alarm"));
Assert.That(alarmCount, Is.EqualTo(Defaults.AutoScaling.Count * numAsgs));
var approxExpectedPerStack = (float)totalResources / numStacks;
foreach (var(_, count) in resourceCountsByStack)
{
Assert.That(count, Is.Not.GreaterThan(approxExpectedPerStack * 1.2));
Assert.That(count, Is.Not.LessThan(approxExpectedPerStack * 0.8));
}
}
public async Task ContinuesWhenAlarmGenerationFailsForOneAlertingGroup()
{
// arrange
var config1 = new AlertingGroup()
{
Name = "group-1",
AlarmNameSuffix = "suffix-1",
Targets = new List <AlertTarget>()
{
new AlertEmail("*****@*****.**")
},
Services = new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1",
Options = new AutoScalingResourceConfig()
{
// this will trigger the failure
InstanceCountIncreaseDelayMinutes = 5
}
}
}
}
}
};
var config2 = new AlertingGroup()
{
Name = "group-2",
AlarmNameSuffix = "suffix-2",
Targets = new List <AlertTarget>()
{
new AlertEmail("*****@*****.**")
},
Services = new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-2"
}
}
}
}
};
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
config1, config2
}
});
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
},
new AutoScalingGroup()
{
AutoScalingGroupName = "group-2",
DesiredCapacity = 10
}
});
ioc.GetMock <IAmazonCloudWatch>()
.Setup(c => c.GetMetricStatisticsAsync(It.IsAny <GetMetricStatisticsRequest>(),
It.IsAny <CancellationToken>()))
.ThrowsAsync(new Exception("something bad"));
var sut = ioc.Get <AlarmLoaderAndGenerator>();
Exception caught = null;
// act
try
{
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
}
catch (Exception ex)
{
caught = ex;
}
// assert
Assert.That(cloudformation.StackWasDeployed("Watchman-group-1"), Is.EqualTo(false));
Assert.That(cloudformation.StackWasDeployed("Watchman-group-2"), Is.EqualTo(true));
Assert.That(caught, Is.Not.Null);
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = "first-sqs-queue"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue",
"first-sqs-queue_error"
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByQueue = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("QueueName");
Assert.That(alarmsByQueue.ContainsKey("first-sqs-queue"), Is.True);
var alarmsForQueue = alarmsByQueue["first-sqs-queue"];
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["AlarmName"].Value <string>().Contains("NumberOfVisibleMessages") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 100 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Average" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
Assert.That(alarmsForQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["AlarmName"].Value <string>().Contains("AgeOfOldestMessage") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 600 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Maximum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
var alarmsForErrorQueue = alarmsByQueue["first-sqs-queue_error"];
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateNumberOfMessagesVisible" &&
alarm.Properties["AlarmName"].Value <string>().Contains("NumberOfVisibleMessages_Error") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 10 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Average" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
Assert.That(alarmsForErrorQueue.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ApproximateAgeOfOldestMessage" &&
alarm.Properties["AlarmName"].Value <string>().Contains("AgeOfOldestMessage_Error") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 600 &&
alarm.Properties["Period"].Value <int>() == 60 * 5 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Maximum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.Sqs &&
alarm.Properties["TreatMissingData"].Value <string>() == TreatMissingDataConstants.Missing
)
);
}
public async Task CanOptOut()
{
// arrange
var config = new WatchmanConfiguration()
{
AlertingGroups = new List <AlertingGroup>()
{
new AlertingGroup()
{
Name = "group-with-description",
AlarmNameSuffix = "group-suffix-1",
Services = new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
// to test we can opt out at resource level
{ "LatencyHigh", new AlarmValues(enabled: false) },
// to test we can revert an opt-out at service level
{ "UnHealthyHostCountHigh", new AlarmValues(enabled: true) }
}
}
},
Values = new Dictionary <string, AlarmValues>()
{
// test we can opt out at service level
{ "Http5xxErrorsHigh", new AlarmValues(enabled: false) },
// setup for above (we can revert an opt-out at service level)
{ "UnHealthyHostCountHigh", new AlarmValues(enabled: false) }
}
}
}
}
}
};
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = fakeCloudFormation
.Stack("Watchman-group-with-description")
.AlarmsByDimension("LoadBalancerName");
var alarmsForElb1 = alarms["elb-1"];
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("LatencyHigh")
), Is.False);
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("Http5xxErrorsHigh")
), Is.False);
Assert.That(alarmsForElb1.Any(
alarm => alarm.Properties["AlarmName"].ToString().Contains("UnHealthyHostCountHigh")
), Is.True);
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
Dax = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >
{
new ResourceThresholds <ResourceConfig>
{
Name = "first-dax-cluster",
Values = new Dictionary <string, AlarmValues>
{
{ "CPUUtilizationHigh", new AlarmValues(10) }
}
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDAX>().HasClusters(new[]
{
new Cluster
{
ClusterName = "first-dax-cluster",
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByCluster = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("ClusterId");
Assert.That(alarmsByCluster.ContainsKey("first-dax-cluster"), Is.True);
var alarms = alarmsByCluster["first-dax-cluster"];
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "CPUUtilization" &&
alarm.Properties["AlarmName"].Value <string>().Contains("CPUUtilizationHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 10 &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Maximum" &&
alarm.Properties["Namespace"].Value <string>() == "AWS/DAX" &&
alarm.Properties["TreatMissingData"].Value <string>() == "missing"
)
);
}
