Exemplo n.º 1
0
        public async Task CanUseExtendedStatisticsForResource()
        {
            // arrange
            var config = ConfigHelper.CreateBasicConfiguration(
                "test",
                "group-suffix",
                new AlertingGroupServices()
            {
                Elb = new AwsServiceAlarms <ResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <ResourceConfig> >()
                    {
                        new ResourceThresholds <ResourceConfig>()
                        {
                            Name   = "elb-1",
                            Values = new Dictionary <string, AlarmValues>()
                            {
                                {
                                    "LatencyHigh", new AlarmValues(extendedStatistic: "p97")
                                }
                            }
                        }
                    }
                }
            }
                );

            var fakeCloudFormation = new FakeCloudFormation();
            var ioc = new TestingIocBootstrapper()
                      .WithCloudFormation(fakeCloudFormation.Instance)
                      .WithConfig(config);

            ioc.GetMock <IAmazonElasticLoadBalancing>().DescribeReturnsLoadBalancers(new[]
            {
                new LoadBalancerDescription()
                {
                    LoadBalancerName = "elb-1"
                }
            });

            var sut = ioc.Get <AlarmLoaderAndGenerator>();

            // act

            await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // assert

            var alarmsByElb = fakeCloudFormation
                              .Stack("Watchman-test")
                              .AlarmsByDimension("LoadBalancerName");

            var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));

            Assert.That(alarm, Is.Not.Null);
            Assert.That(alarm.Properties.ContainsKey("Statistic"), Is.False);
            Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97"));
        }
        public async Task DoesDeployMultipleStacksIfSelected()
        {
            // first create a stack which has a resource

            var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
                                                               new AlertingGroupServices()
            {
                AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
                    {
                        new ResourceThresholds <AutoScalingResourceConfig>()
                        {
                            Name = "group-1"
                        }
                    }
                }
            },
                                                               2);

            var cloudformation = new FakeCloudFormation();

            var firstTestContext = new TestingIocBootstrapper()
                                   .WithCloudFormation(cloudformation.Instance)
                                   .WithConfig(config);

            firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
            {
                new AutoScalingGroup()
                {
                    AutoScalingGroupName = "group-1",
                    DesiredCapacity      = 40
                }
            });

            await firstTestContext.Get <AlarmLoaderAndGenerator>()
            .LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // check it got deployed

            var stack = cloudformation
                        .Stack("Watchman-test");

            Assert.That(stack, Is.Not.Null);
            Assert.That(stack.Resources
                        .Values
                        .Where(r => r.Type == "AWS::CloudWatch::Alarm")
                        .Count, Is.GreaterThan(0));

            var stack2 = cloudformation
                         .Stack("Watchman-test-1");

            Assert.That(stack2, Is.Not.Null);
            Assert.That(stack2.Resources
                        .Values
                        .Where(r => r.Type == "AWS::CloudWatch::Alarm")
                        .Count, Is.GreaterThan(0));
        }
Exemplo n.º 3
0
        public async Task DoesNotDeployNewEmptyStack()
        {
            // arrange
            var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
                                                               new AlertingGroupServices()
            {
                AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
                    {
                        new ResourceThresholds <AutoScalingResourceConfig>()
                        {
                            Name = "non-existent-resource"
                        }
                    }
                }
            });

            var cloudformation = new FakeCloudFormation();
            var ioc            = new TestingIocBootstrapper()
                                 .WithCloudFormation(cloudformation.Instance)
                                 .WithConfig(config);

            ioc.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);


            var now = DateTime.Parse("2018-01-26");

            ioc.GetMock <ICurrentTimeProvider>()
            .Setup(f => f.UtcNow)
            .Returns(now);


            var sut = ioc.Get <AlarmLoaderAndGenerator>();

            // act

            await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // assert

            var stack = cloudformation
                        .Stack("Watchman-test");

            Assert.That(stack, Is.Null);
        }
Exemplo n.º 4
0
        public async Task StacksAreNotListedMultipleTimes()
        {
            var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
                                                               new AlertingGroupServices()
            {
                AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
                    {
                        new ResourceThresholds <AutoScalingResourceConfig>()
                        {
                            Pattern = ".*"
                        }
                    }
                }
            },
                                                               numberOfCloudFormationStacks: 5);

            var cloudFormation = new FakeCloudFormation();

            var context = new TestingIocBootstrapper()
                          .WithCloudFormation(cloudFormation.Instance)
                          .WithConfig(config);

            context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
            {
                new AutoScalingGroup()
                {
                    AutoScalingGroupName = $"group-asg",
                    DesiredCapacity      = 40
                }
            });

            await context.Get <AlarmLoaderAndGenerator>()
            .LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            var stacks = cloudFormation.Stacks();

            Assert.That(stacks.Count, Is.GreaterThan(1));
            Assert.That(cloudFormation.CallsToListStacks, Is.EqualTo(1));
        }
Exemplo n.º 5
0
        public async Task DoesDeployEmptyStackIfAlreadyPresentButResourcesNoLongerExist()
        {
            // first create a stack which has a resource

            var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
                                                               new AlertingGroupServices()
            {
                AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
                    {
                        new ResourceThresholds <AutoScalingResourceConfig>()
                        {
                            Name = "group-1"
                        }
                    }
                }
            });

            var cloudformation = new FakeCloudFormation();

            var firstTestContext = new TestingIocBootstrapper()
                                   .WithCloudFormation(cloudformation.Instance)
                                   .WithConfig(config);

            firstTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new[]
            {
                new AutoScalingGroup()
                {
                    AutoScalingGroupName = "group-1",
                    DesiredCapacity      = 40
                }
            });

            await firstTestContext.Get <AlarmLoaderAndGenerator>()
            .LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // check it got deployed

            var stack = cloudformation
                        .Stack("Watchman-test");

            Assert.That(stack, Is.Not.Null);
            Assert.That(stack.Resources
                        .Values
                        .Where(r => r.Type == "AWS::CloudWatch::Alarm")
                        .Count, Is.GreaterThan(0));

            var secondTestContext = new TestingIocBootstrapper()
                                    .WithCloudFormation(cloudformation.Instance)
                                    .WithConfig(config);

            // no matching resource so the next run results in an empty stack

            secondTestContext.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(new AutoScalingGroup[0]);

            await secondTestContext
            .Get <AlarmLoaderAndGenerator>()
            .LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            stack = cloudformation
                    .Stack("Watchman-test");

            //check we deployed an empty stack and deleted the redundant resources
            Assert.That(stack, Is.Not.Null);
            Assert.That(stack.Resources.Any(x => x.Value.Type == "AWS::CloudWatch::Alarm"), Is.False);
        }
Exemplo n.º 6
0
        public async Task AlarmsAreDistributedEvenlyAcrossStacks()
        {
            const int numStacks = 10;
            const int numAsgs   = 100;

            var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
                                                               new AlertingGroupServices()
            {
                AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
                    {
                        new ResourceThresholds <AutoScalingResourceConfig>()
                        {
                            Pattern = ".*"
                        }
                    }
                }
            },
                                                               numberOfCloudFormationStacks: numStacks);

            var cloudFormation = new FakeCloudFormation();

            var context = new TestingIocBootstrapper()
                          .WithCloudFormation(cloudFormation.Instance)
                          .WithConfig(config);

            var lotsOfAsgs = Enumerable.Range(0, numAsgs).Select(r =>

                                                                 new AutoScalingGroup()
            {
                AutoScalingGroupName = $"group-{r}",
                DesiredCapacity      = 40
            }
                                                                 )
                             .ToArray();

            context.GetMock <IAmazonAutoScaling>().HasAutoScalingGroups(lotsOfAsgs);

            try
            {
                await context.Get <AlarmLoaderAndGenerator>()
                .LoadAndGenerateAlarms(RunMode.GenerateAlarms);
            }
            catch
            {
                // ignore
            }

            var stacks = cloudFormation.Stacks();

            Assert.That(stacks.Count, Is.EqualTo(numStacks));

            var resourceCountsByStack = stacks.Select(s => (s.name, s.template.Resources.Count)).ToArray();

            var totalResources = stacks.Sum(s => s.template.Resources.Count);

            var alarmCount = stacks
                             .Sum(s => s.template.Resources.Count(r => r.Value.Type == "AWS::CloudWatch::Alarm"));

            Assert.That(alarmCount, Is.EqualTo(Defaults.AutoScaling.Count * numAsgs));

            var approxExpectedPerStack = (float)totalResources / numStacks;

            foreach (var(_, count) in resourceCountsByStack)
            {
                Assert.That(count, Is.Not.GreaterThan(approxExpectedPerStack * 1.2));
                Assert.That(count, Is.Not.LessThan(approxExpectedPerStack * 0.8));
            }
        }
Exemplo n.º 7
0
        public async Task AlarmCreatedWithCorrectProperties()
        {
            // arrange
            var config = ConfigHelper.CreateBasicConfiguration(
                "test",
                "group-suffix",
                new AlertingGroupServices()
            {
                DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
                    {
                        new ResourceThresholds <DynamoResourceConfig>()
                        {
                            Name = "first-dynamo-table"
                        }
                    }
                },
                Lambda = new AwsServiceAlarms <ResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <ResourceConfig> >()
                    {
                        new ResourceThresholds <ResourceConfig>()
                        {
                            Name = "first-lambda-function"
                        }
                    }
                }
            }
                );

            var cloudformation = new FakeCloudFormation();
            var ioc            = new TestingIocBootstrapper()
                                 .WithCloudFormation(cloudformation.Instance)
                                 .WithConfig(config);

            ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
            {
                new TableDescription()
                {
                    TableName             = "first-dynamo-table",
                    ProvisionedThroughput = new ProvisionedThroughputDescription()
                    {
                        ReadCapacityUnits  = 100,
                        WriteCapacityUnits = 200
                    }
                }
            });

            ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
            {
                new FunctionConfiguration()
                {
                    FunctionName = "first-lambda-function"
                }
            });

            var sut = ioc.Get <AlarmLoaderAndGenerator>();

            // act

            await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // assert

            var alarmsByTable = cloudformation
                                .Stack("Watchman-test")
                                .AlarmsByDimension("TableName");

            Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);

            var alarmsByFunction = cloudformation
                                   .Stack("Watchman-test")
                                   .AlarmsByDimension("FunctionName");

            Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True);
        }
Exemplo n.º 8
0
        public async Task AlarmCreatedWithCorrectProperties()
        {
            // arrange

            var fakeStackDeployer = new FakeStackDeployer();

            var dynamoClient = FakeAwsClients.CreateDynamoClientForTables(new[]
            {
                new TableDescription()
                {
                    TableName             = "first-dynamo-table",
                    ProvisionedThroughput = new ProvisionedThroughputDescription()
                    {
                        ReadCapacityUnits  = 100,
                        WriteCapacityUnits = 200
                    }
                }
            });

            var lambdaClient = FakeAwsClients.CreateLambdaClientForFunctions(new[]
            {
                new FunctionConfiguration()
                {
                    FunctionName = "first-lambda-function"
                }
            });

            var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true));

            var config = ConfigHelper.CreateBasicConfiguration(
                "test",
                "group-suffix",
                new AlertingGroupServices()
            {
                DynamoDb = new AwsServiceAlarms <ResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <ResourceConfig> >()
                    {
                        new ResourceThresholds <ResourceConfig>()
                        {
                            Name = "first-dynamo-table"
                        }
                    }
                },
                Lambda = new AwsServiceAlarms <ResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <ResourceConfig> >()
                    {
                        new ResourceThresholds <ResourceConfig>()
                        {
                            Name = "first-lambda-function"
                        }
                    }
                }
            }
                );

            var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator);

            sutBuilder.AddService(
                new TableDescriptionSource(dynamoClient),
                new DynamoDbDataProvider(),
                new DynamoDbDataProvider(),
                WatchmanServiceConfigurationMapper.MapDynamoDb
                );

            sutBuilder.AddService(
                new LambdaSource(lambdaClient),
                new LambdaAlarmDataProvider(),
                new LambdaAlarmDataProvider(),
                WatchmanServiceConfigurationMapper.MapLambda
                );

            var sut = sutBuilder.Build();

            // act

            await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // assert

            var alarmsByTable = fakeStackDeployer
                                .Stack("Watchman-test")
                                .AlarmsByDimension("TableName");

            Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);

            var alarmsByFunction = fakeStackDeployer
                                   .Stack("Watchman-test")
                                   .AlarmsByDimension("FunctionName");

            Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True);
        }
Exemplo n.º 9
0
        public async Task CanSetCustomAlarmDescriptionForDifferentServices()
        {
            // arrange
            var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
                                                               new AlertingGroupServices()
            {
                DynamoDb = new AwsServiceAlarms <DynamoResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <DynamoResourceConfig> >()
                    {
                        new ResourceThresholds <DynamoResourceConfig>()
                        {
                            Pattern     = ".*",
                            Description = "custom dynamo text"
                        }
                    }
                },
                Lambda = new AwsServiceAlarms <ResourceConfig>
                {
                    Resources = new List <ResourceThresholds <ResourceConfig> >()
                    {
                        new ResourceThresholds <ResourceConfig>()
                        {
                            Pattern     = ".*",
                            Description = "custom lambda text"
                        }
                    }
                },
                Sqs = new AwsServiceAlarms <SqsResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <SqsResourceConfig> >()
                    {
                        new ResourceThresholds <SqsResourceConfig>()
                        {
                            Pattern     = ".*",
                            Description = "custom sqs text"
                        }
                    }
                }
            });

            var cloudFormation = new FakeCloudFormation();
            var ioc            = new TestingIocBootstrapper()
                                 .WithCloudFormation(cloudFormation.Instance)
                                 .WithConfig(config);

            ioc.GetMock <IAmazonDynamoDB>().HasDynamoTables(new[]
            {
                new TableDescription()
                {
                    TableName             = "first-dynamo-table",
                    ProvisionedThroughput = new ProvisionedThroughputDescription()
                    {
                        ReadCapacityUnits  = 10,
                        WriteCapacityUnits = 10
                    }
                }
            });

            ioc.GetMock <IAmazonCloudWatch>().HasSqsQueues(new[]
            {
                "first-sqs-queue",
                "first-sqs-queue_error"
            });

            ioc.GetMock <IAmazonLambda>().HasLambdaFunctions(new[]
            {
                new FunctionConfiguration  {
                    FunctionName = "first-lambda"
                }
            });

            var sut = ioc.Get <AlarmLoaderAndGenerator>();

            // act

            await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // assert

            Assert.That(cloudFormation.StacksDeployed, Is.EqualTo(1));

            var alarmsByNamespace = cloudFormation.Stacks()
                                    .Single()
                                    .template.AlarmsByNamespace();

            VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.Lambda], "custom lambda text");
            VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.DynamoDb], "custom dynamo text");
            VerifyAlarmDescriptions(alarmsByNamespace[AwsNamespace.Sqs], "custom sqs text");
        }
Exemplo n.º 10
0
        public async Task CanSetExtendedStatisticAtResourceOrServiceLevel()
        {
            // arrange

            var fakeStackDeployer = new FakeStackDeployer();

            var elbClient = FakeAwsClients.CreateElbClientForLoadBalancers(new[]
            {
                new LoadBalancerDescription()
                {
                    LoadBalancerName = "elb-1"
                },
                new LoadBalancerDescription()
                {
                    LoadBalancerName = "elb-2"
                }
            });


            var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true));

            var config = ConfigHelper.CreateBasicConfiguration(
                "test",
                "group-suffix",
                new AlertingGroupServices()
            {
                Elb = new AwsServiceAlarms <ResourceConfig>()
                {
                    Resources = new List <ResourceThresholds <ResourceConfig> >()
                    {
                        new ResourceThresholds <ResourceConfig>()
                        {
                            Name   = "elb-1",
                            Values = new Dictionary <string, AlarmValues>()
                            {
                                {
                                    "LatencyHigh", new AlarmValues(null, null, "p97")
                                }
                            }
                        },

                        new ResourceThresholds <ResourceConfig>()
                        {
                            Name = "elb-2"
                        }
                    },

                    // set default for whole group
                    Values = new Dictionary <string, AlarmValues>()
                    {
                        {
                            "LatencyHigh", new AlarmValues(null, null, "p99")
                        }
                    }
                }
            }
                );

            var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator);

            sutBuilder.AddService(
                new ElbSource(elbClient),
                new ElbAlarmDataProvider(),
                new ElbAlarmDataProvider(),
                WatchmanServiceConfigurationMapper.MapElb
                );

            var sut = sutBuilder.Build();

            // act

            await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);

            // assert

            var alarmsByElb = fakeStackDeployer
                              .Stack("Watchman-test")
                              .AlarmsByDimension("LoadBalancerName");

            // should take override
            var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));

            Assert.That(alarm, Is.Not.Null);
            Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97"));

            // should take default of p99
            var alarm2 = alarmsByElb["elb-2"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));

            Assert.That(alarm2, Is.Not.Null);
            Assert.That(alarm2.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p99"));
        }