public async Task CanUseExtendedStatisticsForResource() { // arrange var config = ConfigHelper.CreateBasicConfiguration( "test", "group-suffix", new AlertingGroupServices() { Elb = new AwsServiceAlarms <ResourceConfig>() { Resources = new List <ResourceThresholds <ResourceConfig> >() { new ResourceThresholds <ResourceConfig>() { Name = "elb-1", Values = new Dictionary <string, AlarmValues>() { { "LatencyHigh", new AlarmValues(extendedStatistic: "p97") } } } } } } ); var fakeCloudFormation = new FakeCloudFormation(); var ioc = new TestingIocBootstrapper() .WithCloudFormation(fakeCloudFormation.Instance) .WithConfig(config); ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[] { new LoadBalancerDescription() { LoadBalancerName = "elb-1" } }); var sut = ioc.Get <AlarmLoaderAndGenerator>(); // act await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms); // assert var alarmsByElb = fakeCloudFormation .Stack("Watchman-test") .AlarmsByDimension("LoadBalancerName"); var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh")); Assert.That(alarm, Is.Not.Null); Assert.That(alarm.Properties.ContainsKey("Statistic"), Is.False); Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97")); }
public async Task DoesDeployMultipleStacksIfSelected() { // first create a stack which has a resource var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices() { AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>() { Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >() { new ResourceThresholds <AutoScalingResourceConfig>() { Name = "group-1" } } } }, 2); var cloudformation = new FakeCloudFormation(); var firstTestContext = new TestingIocBootstrapper() .WithCloudFormation(cloudformation.Instance) .WithConfig(config); firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[] { new AutoScalingGroup() { AutoScalingGroupName = "group-1", DesiredCapacity = 40 } }); await firstTestContext.Get <AlarmLoaderAndGenerator>() .LoadAndGenerateAlarms(RunMode.GenerateAlarms); // check it got deployed var stack = cloudformation .Stack("Watchman-test"); Assert.That(stack, Is.Not.Null); Assert.That(stack.Resources .Values .Where(r => r.Type == "AWS::CloudWatch::Alarm") .Count, Is.GreaterThan(0)); var stack2 = cloudformation .Stack("Watchman-test-1"); Assert.That(stack2, Is.Not.Null); Assert.That(stack2.Resources .Values .Where(r => r.Type == "AWS::CloudWatch::Alarm") .Count, Is.GreaterThan(0)); }
public async Task DoesNotDeployNewEmptyStack() { // arrange var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices() { AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>() { Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >() { new ResourceThresholds <AutoScalingResourceConfig>() { Name = "non-existent-resource" } } } }); var cloudformation = new FakeCloudFormation(); var ioc = new TestingIocBootstrapper() .WithCloudFormation(cloudformation.Instance) .WithConfig(config); ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]); var now = DateTime.Parse("2018-01-26"); ioc.GetMock <ICurrentTimeProvider>() .Setup(f => f.UtcNow) .Returns(now); var sut = ioc.Get <AlarmLoaderAndGenerator>(); // act await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms); // assert var stack = cloudformation .Stack("Watchman-test"); Assert.That(stack, Is.Null); }
public async Task StacksAreNotListedMultipleTimes() { var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices() { AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>() { Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >() { new ResourceThresholds <AutoScalingResourceConfig>() { Pattern = ".*" } } } }, numberOfCloudFormationStacks: 5); var cloudFormation = new FakeCloudFormation(); var context = new TestingIocBootstrapper() .WithCloudFormation(cloudFormation.Instance) .WithConfig(config); context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[] { new AutoScalingGroup() { AutoScalingGroupName = $"group-asg", DesiredCapacity = 40 } }); await context.Get <AlarmLoaderAndGenerator>() .LoadAndGenerateAlarms(RunMode.GenerateAlarms); var stacks = cloudFormation.Stacks(); Assert.That(stacks.Count, Is.GreaterThan(1)); Assert.That(cloudFormation.CallsToListStacks, Is.EqualTo(1)); }
public async Task DoesDeployEmptyStackIfAlreadyPresentButResourcesNoLongerExist() { // first create a stack which has a resource var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices() { AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>() { Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >() { new ResourceThresholds <AutoScalingResourceConfig>() { Name = "group-1" } } } }); var cloudformation = new FakeCloudFormation(); var firstTestContext = new TestingIocBootstrapper() .WithCloudFormation(cloudformation.Instance) .WithConfig(config); firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[] { new AutoScalingGroup() { AutoScalingGroupName = "group-1", DesiredCapacity = 40 } }); await firstTestContext.Get <AlarmLoaderAndGenerator>() .LoadAndGenerateAlarms(RunMode.GenerateAlarms); // check it got deployed var stack = cloudformation .Stack("Watchman-test"); Assert.That(stack, Is.Not.Null); Assert.That(stack.Resources .Values .Where(r => r.Type == "AWS::CloudWatch::Alarm") .Count, Is.GreaterThan(0)); var secondTestContext = new TestingIocBootstrapper() .WithCloudFormation(cloudformation.Instance) .WithConfig(config); // no matching resource so the next run results in an empty stack secondTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]); await secondTestContext .Get <AlarmLoaderAndGenerator>() .LoadAndGenerateAlarms(RunMode.GenerateAlarms); stack = cloudformation .Stack("Watchman-test"); //check we deployed an empty stack and deleted the redundant resources Assert.That(stack, Is.Not.Null); Assert.That(stack.Resources.Any(x => x.Value.Type == "AWS::CloudWatch::Alarm"), Is.False); }
public async Task AlarmsAreDistributedEvenlyAcrossStacks() { const int numStacks = 10; const int numAsgs = 100; var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices() { AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>() { Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >() { new ResourceThresholds <AutoScalingResourceConfig>() { Pattern = ".*" } } } }, numberOfCloudFormationStacks: numStacks); var cloudFormation = new FakeCloudFormation(); var context = new TestingIocBootstrapper() .WithCloudFormation(cloudFormation.Instance) .WithConfig(config); var lotsOfAsgs = Enumerable.Range(0, numAsgs).Select(r => new AutoScalingGroup() { AutoScalingGroupName = $"group-{r}", DesiredCapacity = 40 } ) .ToArray(); context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(lotsOfAsgs); try { await context.Get <AlarmLoaderAndGenerator>() .LoadAndGenerateAlarms(RunMode.GenerateAlarms); } catch { // ignore } var stacks = cloudFormation.Stacks(); Assert.That(stacks.Count, Is.EqualTo(numStacks)); var resourceCountsByStack = stacks.Select(s => (s.name, s.template.Resources.Count)).ToArray(); var totalResources = stacks.Sum(s => s.template.Resources.Count); var alarmCount = stacks .Sum(s => s.template.Resources.Count(r => r.Value.Type == "AWS::CloudWatch::Alarm")); Assert.That(alarmCount, Is.EqualTo(Defaults.AutoScaling.Count * numAsgs)); var approxExpectedPerStack = (float)totalResources / numStacks; foreach (var(_, count) in resourceCountsByStack) { Assert.That(count, Is.Not.GreaterThan(approxExpectedPerStack * 1.2)); Assert.That(count, Is.Not.LessThan(approxExpectedPerStack * 0.8)); } }
public async Task AlarmCreatedWithCorrectProperties() { // arrange var config = ConfigHelper.CreateBasicConfiguration( "test", "group-suffix", new AlertingGroupServices() { DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>() { Resources = new List <ResourceThresholds <DynamoResourceConfig> >() { new ResourceThresholds <DynamoResourceConfig>() { Name = "first-dynamo-table" } } }, Lambda = new AwsServiceAlarms <ResourceConfig>() { Resources = new List <ResourceThresholds <ResourceConfig> >() { new ResourceThresholds <ResourceConfig>() { Name = "first-lambda-function" } } } } ); var cloudformation = new FakeCloudFormation(); var ioc = new TestingIocBootstrapper() .WithCloudFormation(cloudformation.Instance) .WithConfig(config); ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[] { new TableDescription() { TableName = "first-dynamo-table", ProvisionedThroughput = new ProvisionedThroughputDescription() { ReadCapacityUnits = 100, WriteCapacityUnits = 200 } } }); ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[] { new FunctionConfiguration() { FunctionName = "first-lambda-function" } }); var sut = ioc.Get <AlarmLoaderAndGenerator>(); // act await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms); // assert var alarmsByTable = cloudformation .Stack("Watchman-test") .AlarmsByDimension("TableName"); Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True); var alarmsByFunction = cloudformation .Stack("Watchman-test") .AlarmsByDimension("FunctionName"); Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True); }
public async Task AlarmCreatedWithCorrectProperties() { // arrange var fakeStackDeployer = new FakeStackDeployer(); var dynamoClient = FakeAwsClients.CreateDynamoClientForTables(new[] { new TableDescription() { TableName = "first-dynamo-table", ProvisionedThroughput = new ProvisionedThroughputDescription() { ReadCapacityUnits = 100, WriteCapacityUnits = 200 } } }); var lambdaClient = FakeAwsClients.CreateLambdaClientForFunctions(new[] { new FunctionConfiguration() { FunctionName = "first-lambda-function" } }); var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true)); var config = ConfigHelper.CreateBasicConfiguration( "test", "group-suffix", new AlertingGroupServices() { DynamoDb = new AwsServiceAlarms <ResourceConfig>() { Resources = new List <ResourceThresholds <ResourceConfig> >() { new ResourceThresholds <ResourceConfig>() { Name = "first-dynamo-table" } } }, Lambda = new AwsServiceAlarms <ResourceConfig>() { Resources = new List <ResourceThresholds <ResourceConfig> >() { new ResourceThresholds <ResourceConfig>() { Name = "first-lambda-function" } } } } ); var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator); sutBuilder.AddService( new TableDescriptionSource(dynamoClient), new DynamoDbDataProvider(), new DynamoDbDataProvider(), WatchmanServiceConfigurationMapper.MapDynamoDb ); sutBuilder.AddService( new LambdaSource(lambdaClient), new LambdaAlarmDataProvider(), new LambdaAlarmDataProvider(), WatchmanServiceConfigurationMapper.MapLambda ); var sut = sutBuilder.Build(); // act await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms); // assert var alarmsByTable = fakeStackDeployer .Stack("Watchman-test") .AlarmsByDimension("TableName"); Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True); var alarmsByFunction = fakeStackDeployer .Stack("Watchman-test") .AlarmsByDimension("FunctionName"); Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True); }
public async Task CanSetCustomAlarmDescriptionForDifferentServices() { // arrange var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices() { DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>() { Resources = new List <ResourceThresholds <DynamoResourceConfig> >() { new ResourceThresholds <DynamoResourceConfig>() { Pattern = ".*", Description = "custom dynamo text" } } }, Lambda = new AwsServiceAlarms <ResourceConfig> { Resources = new List <ResourceThresholds <ResourceConfig> >() { new ResourceThresholds <ResourceConfig>() { Pattern = ".*", Description = "custom lambda text" } } }, Sqs = new AwsServiceAlarms <SqsResourceConfig>() { Resources = new List <ResourceThresholds <SqsResourceConfig> >() { new ResourceThresholds <SqsResourceConfig>() { Pattern = ".*", Description = "custom sqs text" } } } }); var cloudFormation = new FakeCloudFormation(); var ioc = new TestingIocBootstrapper() .WithCloudFormation(cloudFormation.Instance) .WithConfig(config); ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[] { new TableDescription() { TableName = "first-dynamo-table", ProvisionedThroughput = new ProvisionedThroughputDescription() { ReadCapacityUnits = 10, WriteCapacityUnits = 10 } } }); ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[] { "first-sqs-queue", "first-sqs-queue_error" }); ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[] { new FunctionConfiguration { FunctionName = "first-lambda" } }); var sut = ioc.Get <AlarmLoaderAndGenerator>(); // act await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms); // assert Assert.That(cloudFormation.StacksDeployed, Is.EqualTo(1)); var alarmsByNamespace = cloudFormation.Stacks() .Single() .template.AlarmsByNamespace(); VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.Lambda], "custom lambda text"); VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.DynamoDb], "custom dynamo text"); VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.Sqs], "custom sqs text"); }
public async Task CanSetExtendedStatisticAtResourceOrServiceLevel() { // arrange var fakeStackDeployer = new FakeStackDeployer(); var elbClient = FakeAwsClients.CreateElbClientForLoadBalancers(new[] { new LoadBalancerDescription() { LoadBalancerName = "elb-1" }, new LoadBalancerDescription() { LoadBalancerName = "elb-2" } }); var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true)); var config = ConfigHelper.CreateBasicConfiguration( "test", "group-suffix", new AlertingGroupServices() { Elb = new AwsServiceAlarms <ResourceConfig>() { Resources = new List <ResourceThresholds <ResourceConfig> >() { new ResourceThresholds <ResourceConfig>() { Name = "elb-1", Values = new Dictionary <string, AlarmValues>() { { "LatencyHigh", new AlarmValues(null, null, "p97") } } }, new ResourceThresholds <ResourceConfig>() { Name = "elb-2" } }, // set default for whole group Values = new Dictionary <string, AlarmValues>() { { "LatencyHigh", new AlarmValues(null, null, "p99") } } } } ); var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator); sutBuilder.AddService( new ElbSource(elbClient), new ElbAlarmDataProvider(), new ElbAlarmDataProvider(), WatchmanServiceConfigurationMapper.MapElb ); var sut = sutBuilder.Build(); // act await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms); // assert var alarmsByElb = fakeStackDeployer .Stack("Watchman-test") .AlarmsByDimension("LoadBalancerName"); // should take override var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh")); Assert.That(alarm, Is.Not.Null); Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97")); // should take default of p99 var alarm2 = alarmsByElb["elb-2"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh")); Assert.That(alarm2, Is.Not.Null); Assert.That(alarm2.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p99")); }