public async Task CanUseExtendedStatisticsForResource()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
{
"LatencyHigh", new AlarmValues(extendedStatistic: "p97")
}
}
}
}
}
}
);
var fakeCloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(fakeCloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByElb = fakeCloudFormation
.Stack("Watchman-test")
.AlarmsByDimension("LoadBalancerName");
var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));
Assert.That(alarm, Is.Not.Null);
Assert.That(alarm.Properties.ContainsKey("Statistic"), Is.False);
Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97"));
}
public async Task DoesDeployMultipleStacksIfSelected()
{
// first create a stack which has a resource
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1"
}
}
}
},
2);
var cloudformation = new FakeCloudFormation();
var firstTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
await firstTestContext.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// check it got deployed
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
var stack2 = cloudformation
.Stack("Watchman-test-1");
Assert.That(stack2, Is.Not.Null);
Assert.That(stack2.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
}
public async Task DoesNotDeployNewEmptyStack()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "non-existent-resource"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);
var now = DateTime.Parse("2018-01-26");
ioc.GetMock <ICurrentTimeProvider>()
.Setup(f => f.UtcNow)
.Returns(now);
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Null);
}
public async Task StacksAreNotListedMultipleTimes()
{
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Pattern = ".*"
}
}
}
},
numberOfCloudFormationStacks: 5);
var cloudFormation = new FakeCloudFormation();
var context = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = $"group-asg",
DesiredCapacity = 40
}
});
await context.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
var stacks = cloudFormation.Stacks();
Assert.That(stacks.Count, Is.GreaterThan(1));
Assert.That(cloudFormation.CallsToListStacks, Is.EqualTo(1));
}
public async Task DoesDeployEmptyStackIfAlreadyPresentButResourcesNoLongerExist()
{
// first create a stack which has a resource
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1"
}
}
}
});
var cloudformation = new FakeCloudFormation();
var firstTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
await firstTestContext.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// check it got deployed
var stack = cloudformation
.Stack("Watchman-test");
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources
.Values
.Where(r => r.Type == "AWS::CloudWatch::Alarm")
.Count, Is.GreaterThan(0));
var secondTestContext = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
// no matching resource so the next run results in an empty stack
secondTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);
await secondTestContext
.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
stack = cloudformation
.Stack("Watchman-test");
//check we deployed an empty stack and deleted the redundant resources
Assert.That(stack, Is.Not.Null);
Assert.That(stack.Resources.Any(x => x.Value.Type == "AWS::CloudWatch::Alarm"), Is.False);
}
public async Task AlarmsAreDistributedEvenlyAcrossStacks()
{
const int numStacks = 10;
const int numAsgs = 100;
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Pattern = ".*"
}
}
}
},
numberOfCloudFormationStacks: numStacks);
var cloudFormation = new FakeCloudFormation();
var context = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
var lotsOfAsgs = Enumerable.Range(0, numAsgs).Select(r =>
new AutoScalingGroup()
{
AutoScalingGroupName = $"group-{r}",
DesiredCapacity = 40
}
)
.ToArray();
context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(lotsOfAsgs);
try
{
await context.Get <AlarmLoaderAndGenerator>()
.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
}
catch
{
// ignore
}
var stacks = cloudFormation.Stacks();
Assert.That(stacks.Count, Is.EqualTo(numStacks));
var resourceCountsByStack = stacks.Select(s => (s.name, s.template.Resources.Count)).ToArray();
var totalResources = stacks.Sum(s => s.template.Resources.Count);
var alarmCount = stacks
.Sum(s => s.template.Resources.Count(r => r.Value.Type == "AWS::CloudWatch::Alarm"));
Assert.That(alarmCount, Is.EqualTo(Defaults.AutoScaling.Count * numAsgs));
var approxExpectedPerStack = (float)totalResources / numStacks;
foreach (var(_, count) in resourceCountsByStack)
{
Assert.That(count, Is.Not.GreaterThan(approxExpectedPerStack * 1.2));
Assert.That(count, Is.Not.LessThan(approxExpectedPerStack * 0.8));
}
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Name = "first-dynamo-table"
}
}
},
Lambda = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-lambda-function"
}
}
}
}
);
var cloudformation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudformation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration()
{
FunctionName = "first-lambda-function"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByTable = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarmsByFunction = cloudformation
.Stack("Watchman-test")
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True);
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var fakeStackDeployer = new FakeStackDeployer();
var dynamoClient = FakeAwsClients.CreateDynamoClientForTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
var lambdaClient = FakeAwsClients.CreateLambdaClientForFunctions(new[]
{
new FunctionConfiguration()
{
FunctionName = "first-lambda-function"
}
});
var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-dynamo-table"
}
}
},
Lambda = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-lambda-function"
}
}
}
}
);
var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator);
sutBuilder.AddService(
new TableDescriptionSource(dynamoClient),
new DynamoDbDataProvider(),
new DynamoDbDataProvider(),
WatchmanServiceConfigurationMapper.MapDynamoDb
);
sutBuilder.AddService(
new LambdaSource(lambdaClient),
new LambdaAlarmDataProvider(),
new LambdaAlarmDataProvider(),
WatchmanServiceConfigurationMapper.MapLambda
);
var sut = sutBuilder.Build();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByTable = fakeStackDeployer
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarmsByFunction = fakeStackDeployer
.Stack("Watchman-test")
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True);
}
public async Task CanSetCustomAlarmDescriptionForDifferentServices()
{
// arrange
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
{
Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
{
new ResourceThresholds <DynamoResourceConfig>()
{
Pattern = ".*",
Description = "custom dynamo text"
}
}
},
Lambda = new AwsServiceAlarms <ResourceConfig>
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Pattern = ".*",
Description = "custom lambda text"
}
}
},
Sqs = new AwsServiceAlarms <SqsResourceConfig>()
{
Resources = new List <ResourceThresholds <SqsResourceConfig> >()
{
new ResourceThresholds <SqsResourceConfig>()
{
Pattern = ".*",
Description = "custom sqs text"
}
}
}
});
var cloudFormation = new FakeCloudFormation();
var ioc = new TestingIocBootstrapper()
.WithCloudFormation(cloudFormation.Instance)
.WithConfig(config);
ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 10,
WriteCapacityUnits = 10
}
}
});
ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
{
"first-sqs-queue",
"first-sqs-queue_error"
});
ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
{
new FunctionConfiguration {
FunctionName = "first-lambda"
}
});
var sut = ioc.Get <AlarmLoaderAndGenerator>();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
Assert.That(cloudFormation.StacksDeployed, Is.EqualTo(1));
var alarmsByNamespace = cloudFormation.Stacks()
.Single()
.template.AlarmsByNamespace();
VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.Lambda], "custom lambda text");
VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.DynamoDb], "custom dynamo text");
VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.Sqs], "custom sqs text");
}
public async Task CanSetExtendedStatisticAtResourceOrServiceLevel()
{
// arrange
var fakeStackDeployer = new FakeStackDeployer();
var elbClient = FakeAwsClients.CreateElbClientForLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
{
"LatencyHigh", new AlarmValues(null, null, "p97")
}
}
},
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-2"
}
},
// set default for whole group
Values = new Dictionary <string, AlarmValues>()
{
{
"LatencyHigh", new AlarmValues(null, null, "p99")
}
}
}
}
);
var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator);
sutBuilder.AddService(
new ElbSource(elbClient),
new ElbAlarmDataProvider(),
new ElbAlarmDataProvider(),
WatchmanServiceConfigurationMapper.MapElb
);
var sut = sutBuilder.Build();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByElb = fakeStackDeployer
.Stack("Watchman-test")
.AlarmsByDimension("LoadBalancerName");
// should take override
var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));
Assert.That(alarm, Is.Not.Null);
Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97"));
// should take default of p99
var alarm2 = alarmsByElb["elb-2"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));
Assert.That(alarm2, Is.Not.Null);
Assert.That(alarm2.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p99"));
}