public async Task UsesDelayedScalingThresholdFromCloudWatch()
{
// arrange
var stack = new FakeStackDeployer();
var autoScalingClient = FakeAwsClients.CreateAutoScalingClientForGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-delay-20"
},
new AutoScalingGroup()
{
AutoScalingGroupName = "group-delay-100"
}
});
var source = new AutoScalingGroupSource(autoScalingClient);
var creator = new CloudFormationAlarmCreator(stack, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-delay-20",
Options = new AutoScalingResourceConfig()
{
InstanceCountIncreaseDelayMinutes = 20
}
},
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-delay-100"
}
},
Options = new AutoScalingResourceConfig()
{
InstanceCountIncreaseDelayMinutes = 100
}
}
});
var now = DateTime.Parse("2018-01-26");
var fakeTime = new Mock <ICurrentTimeProvider>();
fakeTime.Setup(f => f.UtcNow).Returns(now);
var cloudWatch = new Mock <IAmazonCloudWatch>();
SetupCloudWatchDesiredMetric(cloudWatch, 100 * 60, now, "group-delay-100", 90);
SetupCloudWatchDesiredMetric(cloudWatch, 20 * 60, now, "group-delay-20", 80);
var provider = new AutoScalingGroupAlarmDataProvider(cloudWatch.Object, fakeTime.Object);
var sut = IoCHelper.CreateSystemUnderTest(
source,
provider,
provider,
WatchmanServiceConfigurationMapper.MapAutoScaling,
creator,
ConfigHelper.ConfigLoaderFor(config)
);
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = stack
.Stack("Watchman-test")
.AlarmsByDimension("AutoScalingGroupName");
var alarm100 = alarms["group-delay-100"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
var alarm20 = alarms["group-delay-20"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
var defaultAlarmThreshold = 0.5m;
Assert.That((decimal)alarm100.Properties["Threshold"], Is.EqualTo(90 * defaultAlarmThreshold));
Assert.That((decimal)alarm20.Properties["Threshold"], Is.EqualTo(80 * defaultAlarmThreshold));
}
public async Task UsesDesiredInstancesForThresholdByDefault()
{
// arrange
var stack = new FakeStackDeployer();
var autoScalingClient = FakeAwsClients.CreateAutoScalingClientForGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
var source = new AutoScalingGroupSource(autoScalingClient);
var creator = new CloudFormationAlarmCreator(stack, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1"
}
}
}
});
var fakeTime = new Mock <ICurrentTimeProvider>();
var cloudWatch = new Mock <IAmazonCloudWatch>();
var provider = new AutoScalingGroupAlarmDataProvider(cloudWatch.Object, fakeTime.Object);
var sut = IoCHelper.CreateSystemUnderTest(
source,
provider,
provider,
WatchmanServiceConfigurationMapper.MapAutoScaling,
creator,
ConfigHelper.ConfigLoaderFor(config)
);
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarms = stack
.Stack("Watchman-test")
.AlarmsByDimension("AutoScalingGroupName");
var alarm = alarms["group-1"].Single(a => a.Properties["AlarmName"].ToString().Contains("InService"));
var defaultAlarmThreshold = 0.5m;
Assert.That((decimal)alarm.Properties["Threshold"], Is.EqualTo(40 * defaultAlarmThreshold));
}
public async Task TimesSuppliedToCloudWatchAreUtc()
{
// arrange
var stack = new FakeStackDeployer();
var autoScalingClient = FakeAwsClients.CreateAutoScalingClientForGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
}
});
var source = new AutoScalingGroupSource(autoScalingClient);
var creator = new CloudFormationAlarmCreator(stack, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1",
Options = new AutoScalingResourceConfig()
{
InstanceCountIncreaseDelayMinutes = 5
}
}
}
}
});
var cloudWatch = new Mock <IAmazonCloudWatch>();
cloudWatch
.Setup(c => c.GetMetricStatisticsAsync(It.IsAny <GetMetricStatisticsRequest>(),
It.IsAny <CancellationToken>()))
.ReturnsAsync(new GetMetricStatisticsResponse()
{
Datapoints = new List <Datapoint>()
{
new Datapoint()
{
Minimum = 5
}
}
});
var provider = new AutoScalingGroupAlarmDataProvider(cloudWatch.Object, new CurrentTimeProvider());
var sut = IoCHelper.CreateSystemUnderTest(
source,
provider,
provider,
WatchmanServiceConfigurationMapper.MapAutoScaling,
creator,
ConfigHelper.ConfigLoaderFor(config)
);
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
cloudWatch.Verify(x => x.GetMetricStatisticsAsync(
It.Is <GetMetricStatisticsRequest>(
r => r.StartTime.Kind == DateTimeKind.Utc &&
r.EndTime.Kind == DateTimeKind.Utc
), It.IsAny <CancellationToken>())
);
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var fakeStackDeployer = new FakeStackDeployer();
var dynamoClient = FakeAwsClients.CreateDynamoClientForTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
var lambdaClient = FakeAwsClients.CreateLambdaClientForFunctions(new[]
{
new FunctionConfiguration()
{
FunctionName = "first-lambda-function"
}
});
var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-dynamo-table"
}
}
},
Lambda = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-lambda-function"
}
}
}
}
);
var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator);
sutBuilder.AddService(
new TableDescriptionSource(dynamoClient),
new DynamoDbDataProvider(),
new DynamoDbDataProvider(),
WatchmanServiceConfigurationMapper.MapDynamoDb
);
sutBuilder.AddService(
new LambdaSource(lambdaClient),
new LambdaAlarmDataProvider(),
new LambdaAlarmDataProvider(),
WatchmanServiceConfigurationMapper.MapLambda
);
var sut = sutBuilder.Build();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByTable = fakeStackDeployer
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarmsByFunction = fakeStackDeployer
.Stack("Watchman-test")
.AlarmsByDimension("FunctionName");
Assert.That(alarmsByFunction.ContainsKey("first-lambda-function"), Is.True);
}
public async Task ContinuesWhenAlarmGenerationFailsForOneAlertingGroup()
{
// arrange
var stack = new FakeStackDeployer();
var autoScalingClient = FakeAwsClients.CreateAutoScalingClientForGroups(new[]
{
new AutoScalingGroup()
{
AutoScalingGroupName = "group-1",
DesiredCapacity = 40
},
new AutoScalingGroup()
{
AutoScalingGroupName = "group-2",
DesiredCapacity = 10
}
});
var source = new AutoScalingGroupSource(autoScalingClient);
var creator = new CloudFormationAlarmCreator(stack, new ConsoleAlarmLogger(true));
var config1 = new AlertingGroup()
{
Name = "group-1",
AlarmNameSuffix = "suffix-1",
Targets = new List <AlertTarget>()
{
new AlertEmail("*****@*****.**")
},
Services = new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-1",
Options = new AutoScalingResourceConfig()
{
// this will trigger the failure
InstanceCountIncreaseDelayMinutes = 5
}
}
}
}
}
};
var config2 = new AlertingGroup()
{
Name = "group-2",
AlarmNameSuffix = "suffix-2",
Targets = new List <AlertTarget>()
{
new AlertEmail("*****@*****.**")
},
Services = new AlertingGroupServices()
{
AutoScaling = new AwsServiceAlarms <AutoScalingResourceConfig>()
{
Resources = new List <ResourceThresholds <AutoScalingResourceConfig> >()
{
new ResourceThresholds <AutoScalingResourceConfig>()
{
Name = "group-2"
}
}
}
}
};
var cloudWatch = new Mock <IAmazonCloudWatch>();
cloudWatch
.Setup(c => c.GetMetricStatisticsAsync(It.IsAny <GetMetricStatisticsRequest>(),
It.IsAny <CancellationToken>()))
.ThrowsAsync(new Exception("something bad"));
var provider = new AutoScalingGroupAlarmDataProvider(cloudWatch.Object, new CurrentTimeProvider());
var sut = IoCHelper.CreateSystemUnderTest(
source,
provider,
provider,
WatchmanServiceConfigurationMapper.MapAutoScaling,
creator,
ConfigHelper.ConfigLoaderFor(config1, config2)
);
Exception caught = null;
// act
try
{
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
}
catch (Exception ex)
{
caught = ex;
}
// assert
Assert.That(stack.StackWasDeployed("Watchman-group-1"), Is.EqualTo(false));
Assert.That(stack.StackWasDeployed("Watchman-group-2"), Is.EqualTo(true));
Assert.That(caught, Is.Not.Null);
}
public async Task AlarmCreatedWithCorrectProperties()
{
// arrange
var stack = new FakeStackDeployer();
var dynamoClient = FakeAwsClients.CreateDynamoClientForTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 100,
WriteCapacityUnits = 200
}
}
});
var source = new TableDescriptionSource(dynamoClient);
var creator = new CloudFormationAlarmCreator(stack, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix", new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "first-dynamo-table"
}
}
}
});
var sut = IoCHelper.CreateSystemUnderTest(
source,
new DynamoDbDataProvider(),
new DynamoDbDataProvider(),
WatchmanServiceConfigurationMapper.MapDynamoDb,
creator, ConfigHelper.ConfigLoaderFor(config)
);
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
const decimal defaultCapacityThresholdFraction = 0.8m;
const int defaultThrottleThreshold = 2;
var alarmsByTable = stack
.Stack("Watchman-test")
.AlarmsByDimension("TableName");
Assert.That(alarmsByTable.ContainsKey("first-dynamo-table"), Is.True);
var alarms = alarmsByTable["first-dynamo-table"];
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ConsumedReadCapacityUnits" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ConsumedReadCapacityUnitsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 100 * defaultCapacityThresholdFraction &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.DynamoDb
)
);
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ConsumedWriteCapacityUnits" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ConsumedWriteCapacityUnitsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == 200 * defaultCapacityThresholdFraction &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.DynamoDb
)
);
Assert.That(alarms.Exists(
alarm =>
alarm.Properties["MetricName"].Value <string>() == "ThrottledRequests" &&
alarm.Properties["AlarmName"].Value <string>().Contains("ThrottledRequestsHigh") &&
alarm.Properties["AlarmName"].Value <string>().Contains("-group-suffix") &&
alarm.Properties["Threshold"].Value <int>() == defaultThrottleThreshold &&
alarm.Properties["Period"].Value <int>() == 60 &&
alarm.Properties["ComparisonOperator"].Value <string>() == "GreaterThanOrEqualToThreshold" &&
alarm.Properties["Statistic"].Value <string>() == "Sum" &&
alarm.Properties["Namespace"].Value <string>() == AwsNamespace.DynamoDb
)
);
}
public async Task IgnoresNamedEntitiesThatDoNotExist()
{
// arrange
var stack = new Mock <ICloudformationStackDeployer>();
var dynamoClient = FakeAwsClients.CreateDynamoClientForTables(new[]
{
new TableDescription()
{
TableName = "first-dynamo-table",
ProvisionedThroughput = new ProvisionedThroughputDescription()
{
ReadCapacityUnits = 10,
WriteCapacityUnits = 10
}
}
});
var source = new TableDescriptionSource(dynamoClient);
var creator = new CloudFormationAlarmCreator(stack.Object, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration("test", "group-suffix",
new AlertingGroupServices()
{
DynamoDb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "non-existant-table"
}
}
}
});
var sut = IoCHelper.CreateSystemUnderTest(
source,
new DynamoDbDataProvider(),
new DynamoDbDataProvider(),
WatchmanServiceConfigurationMapper.MapDynamoDb,
creator,
ConfigHelper.ConfigLoaderFor(config)
);
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
stack
.Verify(x => x.DeployStack(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <bool>()
), Times.Never);
}
public async Task CanSetExtendedStatisticAtResourceOrServiceLevel()
{
// arrange
var fakeStackDeployer = new FakeStackDeployer();
var elbClient = FakeAwsClients.CreateElbClientForLoadBalancers(new[]
{
new LoadBalancerDescription()
{
LoadBalancerName = "elb-1"
},
new LoadBalancerDescription()
{
LoadBalancerName = "elb-2"
}
});
var creator = new CloudFormationAlarmCreator(fakeStackDeployer, new ConsoleAlarmLogger(true));
var config = ConfigHelper.CreateBasicConfiguration(
"test",
"group-suffix",
new AlertingGroupServices()
{
Elb = new AwsServiceAlarms <ResourceConfig>()
{
Resources = new List <ResourceThresholds <ResourceConfig> >()
{
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-1",
Values = new Dictionary <string, AlarmValues>()
{
{
"LatencyHigh", new AlarmValues(null, null, "p97")
}
}
},
new ResourceThresholds <ResourceConfig>()
{
Name = "elb-2"
}
},
// set default for whole group
Values = new Dictionary <string, AlarmValues>()
{
{
"LatencyHigh", new AlarmValues(null, null, "p99")
}
}
}
}
);
var sutBuilder = new Builder(ConfigHelper.ConfigLoaderFor(config), creator);
sutBuilder.AddService(
new ElbSource(elbClient),
new ElbAlarmDataProvider(),
new ElbAlarmDataProvider(),
WatchmanServiceConfigurationMapper.MapElb
);
var sut = sutBuilder.Build();
// act
await sut.LoadAndGenerateAlarms(RunMode.GenerateAlarms);
// assert
var alarmsByElb = fakeStackDeployer
.Stack("Watchman-test")
.AlarmsByDimension("LoadBalancerName");
// should take override
var alarm = alarmsByElb["elb-1"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));
Assert.That(alarm, Is.Not.Null);
Assert.That(alarm.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p97"));
// should take default of p99
var alarm2 = alarmsByElb["elb-2"].FirstOrDefault(a => a.Properties["AlarmName"].ToString().Contains("LatencyHigh"));
Assert.That(alarm2, Is.Not.Null);
Assert.That(alarm2.Properties["ExtendedStatistic"].ToString(), Is.EqualTo("p99"));
}
