/// <summary> /// Check service health. /// </summary> /// <param name="client">Instance of <see cref="AmazonCloudWatchLogsClient"/> class.</param> /// <returns>Success, RountTripTime.</returns> public static async Task <(bool, double)> CheckServiceReachable(AmazonCloudWatchLogsClient client) { var stopwatch = new Stopwatch(); try { stopwatch.Start(); await client.DescribeLogGroupsAsync(new DescribeLogGroupsRequest { LogGroupNamePrefix = "KinesisTap" }); stopwatch.Stop(); } catch (AmazonCloudWatchLogsException) { stopwatch.Stop(); // Any exception is fine, we are currently only looking to // check if the service is reachable and what is the RTT. return(true, stopwatch.ElapsedMilliseconds); } catch (Exception) { stopwatch.Stop(); return(false, stopwatch.ElapsedMilliseconds); } return(true, stopwatch.ElapsedMilliseconds); }
private async Task <IEnumerable <string> > GetLogGroups(IEnumerable <string> logGroups = null, string nextToken = default) { while (true) { if (logGroups == null) { logGroups = new string[0]; } var request = new DescribeLogGroupsRequest { Limit = 50, LogGroupNamePrefix = _prefix, NextToken = nextToken }; var result = await _cloudWatchLogs.DescribeLogGroupsAsync(request); logGroups = logGroups.Concat(result.LogGroups.Select(logGroup => logGroup.LogGroupName)); if (!string.IsNullOrEmpty(result.NextToken)) { nextToken = result.NextToken; } else { return(logGroups); } } }
// snippet-start:[CloudWatchLogs.dotnetv3.DescribeLogGroupsExample] public static async Task Main() { // Creates a CloudWatch Logs client using the default // user. If you need to work with resources in another // AWS Region than the one defined for the default user, // pass the AWS Region as a parameter to the client constructor. var client = new AmazonCloudWatchLogsClient(); var request = new DescribeLogGroupsRequest { Limit = 5, }; var response = await client.DescribeLogGroupsAsync(request); if (response.LogGroups.Count > 0) { do { response.LogGroups.ForEach(lg => { Console.WriteLine($"{lg.LogGroupName} is associated with the key: {lg.KmsKeyId}."); Console.WriteLine($"Created on: {lg.CreationTime.Date.Date}"); Console.WriteLine($"Date for this group will be stored for: {lg.RetentionInDays} days.\n"); }); }while (response.NextToken is not null); } }
protected virtual void Dispose(bool disposing) { AmazonCloudWatchLogsClient Client = new AmazonCloudWatchLogsClient(Amazon.RegionEndpoint.USWest2); foreach (var logGroupName in LogGroupNameList) { if (!(string.IsNullOrEmpty(logGroupName))) { DescribeLogGroupsResponse describeLogGroupsResponse = Client.DescribeLogGroupsAsync( new DescribeLogGroupsRequest { LogGroupNamePrefix = logGroupName }).Result; if (!(string.IsNullOrEmpty(describeLogGroupsResponse.LogGroups[0].LogGroupName))) { var response = Client.DeleteLogGroupAsync(new DeleteLogGroupRequest { LogGroupName = logGroupName }).Result; } } } }
public IEnumerable <IRow> Read() { if (!_run) { yield break; } do { var response = _client.DescribeLogGroupsAsync().Result; foreach (var logGroup in response.LogGroups) { var row = _rowFactory.Create(); foreach (var field in _context.InputFields) { var name = field.Name.ToLower(); switch (name) { case "arn": row[field] = logGroup.Arn ?? string.Empty; break; case "creationtime": row[field] = logGroup.CreationTime; break; case "kmskeyid": row[field] = logGroup.KmsKeyId ?? string.Empty; break; case "loggroupname": row[field] = logGroup.LogGroupName ?? string.Empty; break; case "metricfiltercount": row[field] = logGroup.MetricFilterCount; break; case "retentionindays": row[field] = logGroup.RetentionInDays.GetValueOrDefault(); break; case "storedbytes": row[field] = logGroup.StoredBytes; break; default: break; } } yield return(row); } _request.NextToken = response.NextToken; } while (!string.IsNullOrEmpty(_request.NextToken)); }
private static async Task <List <SubscriptionConfiguration> > ExpandSubscriptionPattern(SubscriptionConfiguration originalSubscription) { var cw = new AmazonCloudWatchLogsClient(RegionEndpoint.GetBySystemName(originalSubscription.AwsRegion)); var regex = new Regex(originalSubscription.LogGroupPattern, RegexOptions.IgnoreCase); var expandedList = new List <SubscriptionConfiguration>(); var nextToken = (string)null; do { var response = await cw.DescribeLogGroupsAsync(new DescribeLogGroupsRequest { NextToken = nextToken }); if (response.HttpStatusCode != HttpStatusCode.OK) { throw new InvalidOperationException($"Subscription '{originalSubscription.Id}' - " + $"Could not query AWS CloudWatch Logs for list of log groups"); } nextToken = response.NextToken; foreach (var logGroup in response.LogGroups.OrEmpty()) { if (!regex.IsMatch(logGroup.LogGroupName)) { continue; } var clonedSubscription = originalSubscription.Clone(false); clonedSubscription.LogGroupPattern = null; clonedSubscription.LogGroupName = logGroup.LogGroupName; clonedSubscription.Id += "." + ConfigurationIdCleanupRegex.Replace(logGroup.LogGroupName, "_"); expandedList.Add(clonedSubscription); } }while (nextToken.HasValue()); Log.Logger.Information( "Expanded {OriginalId} into {ExpandedCount} subscriptions for log groups: {ExpandedList}", originalSubscription.Id, expandedList.Count, string.Join(", ", expandedList.Select(s => s.LogGroupName))); return(expandedList); }
public async Task DeleteOrphanLogsAsync(bool dryRun, string awsProfile, string awsRegion) { Console.WriteLine(); // initialize AWS profile await InitializeAwsProfile(awsProfile, awsRegion : awsRegion); var logsClient = new AmazonCloudWatchLogsClient(AWSConfigs.RegionEndpoint); // delete orphaned logs var totalLogGroups = 0; var activeLogGroups = 0; var orphanedLogGroups = 0; var skippedLogGroups = 0; await DeleteOrphanLambdaLogsAsync(); await DeleteOrphanApiGatewayLogs(); await DeleteOrphanApiGatewayV2Logs(); if ((orphanedLogGroups > 0) || (skippedLogGroups > 0)) { Console.WriteLine(); } Console.WriteLine($"Found {totalLogGroups:N0} log groups. Active {activeLogGroups:N0}. Orphaned {orphanedLogGroups:N0}. Skipped {skippedLogGroups:N0}."); // local functions async Task DeleteOrphanLambdaLogsAsync() { // list all lambda functions var lambdaClient = new AmazonLambdaClient(AWSConfigs.RegionEndpoint); var request = new ListFunctionsRequest { }; var lambdaLogGroupNames = new HashSet <string>(); do { var response = await lambdaClient.ListFunctionsAsync(request); foreach (var function in response.Functions) { lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}"); } request.Marker = response.NextMarker; } while(request.Marker != null); // list all log groups for lambda functions await DeleteOrphanCloudWatchLogs( "/aws/lambda/", logGroupName => lambdaLogGroupNames.Contains(logGroupName), logGroupName => Regex.IsMatch(logGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$") ); } async Task DeleteOrphanApiGatewayLogs() { // list all API Gateway V1 instances var apiGatewayClient = new AmazonAPIGatewayClient(AWSConfigs.RegionEndpoint); var request = new GetRestApisRequest { }; var apiGatewayGroupNames = new List <string>(); do { var response = await apiGatewayClient.GetRestApisAsync(request); apiGatewayGroupNames.AddRange(response.Items.Select(item => $"API-Gateway-Execution-Logs_{item.Id}/")); request.Position = response.Position; } while(request.Position != null); // list all log groups for API Gateway instances await DeleteOrphanCloudWatchLogs( "API-Gateway-Execution-Logs_", logGroupName => apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)), logGroupName => Regex.IsMatch(logGroupName, @"^API-Gateway-Execution-Logs_[a-zA-Z0-9]+/.+$") ); } async Task DeleteOrphanApiGatewayV2Logs() { // list all API Gateway V2 instances var apiGatewayV2Client = new AmazonApiGatewayV2Client(AWSConfigs.RegionEndpoint); var request = new GetApisRequest { }; var apiGatewayGroupNames = new List <string>(); do { var response = await apiGatewayV2Client.GetApisAsync(request); apiGatewayGroupNames.AddRange(response.Items.Select(item => $"/aws/apigateway/{item.ApiId}/")); request.NextToken = response.NextToken; } while(request.NextToken != null); // list all log groups for API Gateway instances await DeleteOrphanCloudWatchLogs( "/aws/apigateway/", logGroupName => (logGroupName == "/aws/apigateway/welcome") || apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)), logGroupName => Regex.IsMatch(logGroupName, @"^/aws/apigateway/[a-zA-Z0-9]+/.+$") ); } async Task DeleteOrphanCloudWatchLogs(string logGroupPrefix, Func <string, bool> isActiveLogGroup, Func <string, bool> isValidLogGroup) { var describeLogGroupsRequest = new DescribeLogGroupsRequest { LogGroupNamePrefix = logGroupPrefix }; do { var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest); totalLogGroups += describeLogGroupsResponse.LogGroups.Count; foreach (var logGroup in describeLogGroupsResponse.LogGroups) { if (isActiveLogGroup(logGroup.LogGroupName)) { // nothing to do ++activeLogGroups; } else if (isValidLogGroup(logGroup.LogGroupName)) { // attempt to delete log group if (dryRun) { Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)"); ++orphanedLogGroups; } else { try { await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest { LogGroupName = logGroup.LogGroupName }); Console.WriteLine($"* deleted '{logGroup.LogGroupName}'"); ++orphanedLogGroups; } catch { LogError($"could not delete '{logGroup.LogGroupName}'"); ++skippedLogGroups; } } } else { // log group has an invalid name structure; skip it Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'"); ++skippedLogGroups; } } describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken; } while(describeLogGroupsRequest.NextToken != null); } }
public async Task DeleteOrphanLambdaLogsAsync(bool dryRun) { Console.WriteLine(); // list all lambda functions var lambdaClient = new AmazonLambdaClient(); var listFunctionsRequest = new ListFunctionsRequest { }; var lambdaLogGroupNames = new HashSet <string>(); do { var listFunctionsResponse = await lambdaClient.ListFunctionsAsync(listFunctionsRequest); foreach (var function in listFunctionsResponse.Functions) { lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}"); } listFunctionsRequest.Marker = listFunctionsResponse.NextMarker; } while(listFunctionsRequest.Marker != null); // list all log groups for lambda functions var logsClient = new AmazonCloudWatchLogsClient(); var describeLogGroupsRequest = new DescribeLogGroupsRequest { LogGroupNamePrefix = "/aws/lambda/" }; var totalLogGroups = 0; var deletedLogGroups = 0; var skippedLogGroups = 0; do { var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest); totalLogGroups += describeLogGroupsResponse.LogGroups.Count; foreach (var logGroup in describeLogGroupsResponse.LogGroups) { if (lambdaLogGroupNames.Contains(logGroup.LogGroupName)) { // nothing to do } else if (System.Text.RegularExpressions.Regex.IsMatch(logGroup.LogGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$")) { // attempt to delete log group if (dryRun) { Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)"); } else { try { await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest { LogGroupName = logGroup.LogGroupName }); Console.WriteLine($"* deleted '{logGroup.LogGroupName}'"); ++deletedLogGroups; } catch { LogError($"could not delete '{logGroup.LogGroupName}'"); } } } else { // log group has an invalid name structure; skip it Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'"); ++skippedLogGroups; } } describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken; } while(describeLogGroupsRequest.NextToken != null); if ((deletedLogGroups > 0) || (skippedLogGroups > 0)) { Console.WriteLine(); } Console.WriteLine($"Found {totalLogGroups:N0} log groups. Deleted {deletedLogGroups:N0}. Skipped {skippedLogGroups:N0}."); }