/// <summary>
/// Check service health.
/// </summary>
/// <param name="client">Instance of <see cref="AmazonCloudWatchLogsClient"/> class.</param>
/// <returns>Success, RountTripTime.</returns>
public static async Task <(bool, double)> CheckServiceReachable(AmazonCloudWatchLogsClient client)
{
var stopwatch = new Stopwatch();
try
{
stopwatch.Start();
await client.DescribeLogGroupsAsync(new DescribeLogGroupsRequest
{
LogGroupNamePrefix = "KinesisTap"
});
stopwatch.Stop();
}
catch (AmazonCloudWatchLogsException)
{
stopwatch.Stop();
// Any exception is fine, we are currently only looking to
// check if the service is reachable and what is the RTT.
return(true, stopwatch.ElapsedMilliseconds);
}
catch (Exception)
{
stopwatch.Stop();
return(false, stopwatch.ElapsedMilliseconds);
}
return(true, stopwatch.ElapsedMilliseconds);
}
private async Task <IEnumerable <string> > GetLogGroups(IEnumerable <string> logGroups = null, string nextToken = default)
{
while (true)
{
if (logGroups == null)
{
logGroups = new string[0];
}
var request = new DescribeLogGroupsRequest {
Limit = 50, LogGroupNamePrefix = _prefix, NextToken = nextToken
};
var result = await _cloudWatchLogs.DescribeLogGroupsAsync(request);
logGroups = logGroups.Concat(result.LogGroups.Select(logGroup => logGroup.LogGroupName));
if (!string.IsNullOrEmpty(result.NextToken))
{
nextToken = result.NextToken;
}
else
{
return(logGroups);
}
}
}
// snippet-start:[CloudWatchLogs.dotnetv3.DescribeLogGroupsExample]
public static async Task Main()
{
// Creates a CloudWatch Logs client using the default
// user. If you need to work with resources in another
// AWS Region than the one defined for the default user,
// pass the AWS Region as a parameter to the client constructor.
var client = new AmazonCloudWatchLogsClient();
var request = new DescribeLogGroupsRequest
{
Limit = 5,
};
var response = await client.DescribeLogGroupsAsync(request);
if (response.LogGroups.Count > 0)
{
do
{
response.LogGroups.ForEach(lg =>
{
Console.WriteLine($"{lg.LogGroupName} is associated with the key: {lg.KmsKeyId}.");
Console.WriteLine($"Created on: {lg.CreationTime.Date.Date}");
Console.WriteLine($"Date for this group will be stored for: {lg.RetentionInDays} days.\n");
});
}while (response.NextToken is not null);
}
}
protected virtual void Dispose(bool disposing)
{
AmazonCloudWatchLogsClient Client =
new AmazonCloudWatchLogsClient(Amazon.RegionEndpoint.USWest2);
foreach (var logGroupName in LogGroupNameList)
{
if (!(string.IsNullOrEmpty(logGroupName)))
{
DescribeLogGroupsResponse describeLogGroupsResponse = Client.DescribeLogGroupsAsync(
new DescribeLogGroupsRequest
{
LogGroupNamePrefix = logGroupName
}).Result;
if (!(string.IsNullOrEmpty(describeLogGroupsResponse.LogGroups[0].LogGroupName)))
{
var response = Client.DeleteLogGroupAsync(new DeleteLogGroupRequest
{
LogGroupName = logGroupName
}).Result;
}
}
}
}
public IEnumerable <IRow> Read()
{
if (!_run)
{
yield break;
}
do
{
var response = _client.DescribeLogGroupsAsync().Result;
foreach (var logGroup in response.LogGroups)
{
var row = _rowFactory.Create();
foreach (var field in _context.InputFields)
{
var name = field.Name.ToLower();
switch (name)
{
case "arn":
row[field] = logGroup.Arn ?? string.Empty;
break;
case "creationtime":
row[field] = logGroup.CreationTime;
break;
case "kmskeyid":
row[field] = logGroup.KmsKeyId ?? string.Empty;
break;
case "loggroupname":
row[field] = logGroup.LogGroupName ?? string.Empty;
break;
case "metricfiltercount":
row[field] = logGroup.MetricFilterCount;
break;
case "retentionindays":
row[field] = logGroup.RetentionInDays.GetValueOrDefault();
break;
case "storedbytes":
row[field] = logGroup.StoredBytes;
break;
default:
break;
}
}
yield return(row);
}
_request.NextToken = response.NextToken;
} while (!string.IsNullOrEmpty(_request.NextToken));
}
private static async Task <List <SubscriptionConfiguration> > ExpandSubscriptionPattern(SubscriptionConfiguration originalSubscription)
{
var cw = new AmazonCloudWatchLogsClient(RegionEndpoint.GetBySystemName(originalSubscription.AwsRegion));
var regex = new Regex(originalSubscription.LogGroupPattern, RegexOptions.IgnoreCase);
var expandedList = new List <SubscriptionConfiguration>();
var nextToken = (string)null;
do
{
var response = await cw.DescribeLogGroupsAsync(new DescribeLogGroupsRequest { NextToken = nextToken });
if (response.HttpStatusCode != HttpStatusCode.OK)
{
throw new InvalidOperationException($"Subscription '{originalSubscription.Id}' - " +
$"Could not query AWS CloudWatch Logs for list of log groups");
}
nextToken = response.NextToken;
foreach (var logGroup in response.LogGroups.OrEmpty())
{
if (!regex.IsMatch(logGroup.LogGroupName))
{
continue;
}
var clonedSubscription = originalSubscription.Clone(false);
clonedSubscription.LogGroupPattern = null;
clonedSubscription.LogGroupName = logGroup.LogGroupName;
clonedSubscription.Id += "." + ConfigurationIdCleanupRegex.Replace(logGroup.LogGroupName, "_");
expandedList.Add(clonedSubscription);
}
}while (nextToken.HasValue());
Log.Logger.Information(
"Expanded {OriginalId} into {ExpandedCount} subscriptions for log groups: {ExpandedList}",
originalSubscription.Id,
expandedList.Count,
string.Join(", ", expandedList.Select(s => s.LogGroupName)));
return(expandedList);
}
public async Task DeleteOrphanLogsAsync(bool dryRun, string awsProfile, string awsRegion)
{
Console.WriteLine();
// initialize AWS profile
await InitializeAwsProfile(awsProfile, awsRegion : awsRegion);
var logsClient = new AmazonCloudWatchLogsClient(AWSConfigs.RegionEndpoint);
// delete orphaned logs
var totalLogGroups = 0;
var activeLogGroups = 0;
var orphanedLogGroups = 0;
var skippedLogGroups = 0;
await DeleteOrphanLambdaLogsAsync();
await DeleteOrphanApiGatewayLogs();
await DeleteOrphanApiGatewayV2Logs();
if ((orphanedLogGroups > 0) || (skippedLogGroups > 0))
{
Console.WriteLine();
}
Console.WriteLine($"Found {totalLogGroups:N0} log groups. Active {activeLogGroups:N0}. Orphaned {orphanedLogGroups:N0}. Skipped {skippedLogGroups:N0}.");
// local functions
async Task DeleteOrphanLambdaLogsAsync()
{
// list all lambda functions
var lambdaClient = new AmazonLambdaClient(AWSConfigs.RegionEndpoint);
var request = new ListFunctionsRequest {
};
var lambdaLogGroupNames = new HashSet <string>();
do
{
var response = await lambdaClient.ListFunctionsAsync(request);
foreach (var function in response.Functions)
{
lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}");
}
request.Marker = response.NextMarker;
} while(request.Marker != null);
// list all log groups for lambda functions
await DeleteOrphanCloudWatchLogs(
"/aws/lambda/",
logGroupName => lambdaLogGroupNames.Contains(logGroupName),
logGroupName => Regex.IsMatch(logGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$")
);
}
async Task DeleteOrphanApiGatewayLogs()
{
// list all API Gateway V1 instances
var apiGatewayClient = new AmazonAPIGatewayClient(AWSConfigs.RegionEndpoint);
var request = new GetRestApisRequest {
};
var apiGatewayGroupNames = new List <string>();
do
{
var response = await apiGatewayClient.GetRestApisAsync(request);
apiGatewayGroupNames.AddRange(response.Items.Select(item => $"API-Gateway-Execution-Logs_{item.Id}/"));
request.Position = response.Position;
} while(request.Position != null);
// list all log groups for API Gateway instances
await DeleteOrphanCloudWatchLogs(
"API-Gateway-Execution-Logs_",
logGroupName => apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)),
logGroupName => Regex.IsMatch(logGroupName, @"^API-Gateway-Execution-Logs_[a-zA-Z0-9]+/.+$")
);
}
async Task DeleteOrphanApiGatewayV2Logs()
{
// list all API Gateway V2 instances
var apiGatewayV2Client = new AmazonApiGatewayV2Client(AWSConfigs.RegionEndpoint);
var request = new GetApisRequest {
};
var apiGatewayGroupNames = new List <string>();
do
{
var response = await apiGatewayV2Client.GetApisAsync(request);
apiGatewayGroupNames.AddRange(response.Items.Select(item => $"/aws/apigateway/{item.ApiId}/"));
request.NextToken = response.NextToken;
} while(request.NextToken != null);
// list all log groups for API Gateway instances
await DeleteOrphanCloudWatchLogs(
"/aws/apigateway/",
logGroupName => (logGroupName == "/aws/apigateway/welcome") || apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)),
logGroupName => Regex.IsMatch(logGroupName, @"^/aws/apigateway/[a-zA-Z0-9]+/.+$")
);
}
async Task DeleteOrphanCloudWatchLogs(string logGroupPrefix, Func <string, bool> isActiveLogGroup, Func <string, bool> isValidLogGroup)
{
var describeLogGroupsRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = logGroupPrefix
};
do
{
var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest);
totalLogGroups += describeLogGroupsResponse.LogGroups.Count;
foreach (var logGroup in describeLogGroupsResponse.LogGroups)
{
if (isActiveLogGroup(logGroup.LogGroupName))
{
// nothing to do
++activeLogGroups;
}
else if (isValidLogGroup(logGroup.LogGroupName))
{
// attempt to delete log group
if (dryRun)
{
Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)");
++orphanedLogGroups;
}
else
{
try {
await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest {
LogGroupName = logGroup.LogGroupName
});
Console.WriteLine($"* deleted '{logGroup.LogGroupName}'");
++orphanedLogGroups;
} catch {
LogError($"could not delete '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
}
else
{
// log group has an invalid name structure; skip it
Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken;
} while(describeLogGroupsRequest.NextToken != null);
}
}
public async Task DeleteOrphanLambdaLogsAsync(bool dryRun)
{
Console.WriteLine();
// list all lambda functions
var lambdaClient = new AmazonLambdaClient();
var listFunctionsRequest = new ListFunctionsRequest {
};
var lambdaLogGroupNames = new HashSet <string>();
do
{
var listFunctionsResponse = await lambdaClient.ListFunctionsAsync(listFunctionsRequest);
foreach (var function in listFunctionsResponse.Functions)
{
lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}");
}
listFunctionsRequest.Marker = listFunctionsResponse.NextMarker;
} while(listFunctionsRequest.Marker != null);
// list all log groups for lambda functions
var logsClient = new AmazonCloudWatchLogsClient();
var describeLogGroupsRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = "/aws/lambda/"
};
var totalLogGroups = 0;
var deletedLogGroups = 0;
var skippedLogGroups = 0;
do
{
var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest);
totalLogGroups += describeLogGroupsResponse.LogGroups.Count;
foreach (var logGroup in describeLogGroupsResponse.LogGroups)
{
if (lambdaLogGroupNames.Contains(logGroup.LogGroupName))
{
// nothing to do
}
else if (System.Text.RegularExpressions.Regex.IsMatch(logGroup.LogGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$"))
{
// attempt to delete log group
if (dryRun)
{
Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)");
}
else
{
try {
await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest {
LogGroupName = logGroup.LogGroupName
});
Console.WriteLine($"* deleted '{logGroup.LogGroupName}'");
++deletedLogGroups;
} catch {
LogError($"could not delete '{logGroup.LogGroupName}'");
}
}
}
else
{
// log group has an invalid name structure; skip it
Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken;
} while(describeLogGroupsRequest.NextToken != null);
if ((deletedLogGroups > 0) || (skippedLogGroups > 0))
{
Console.WriteLine();
}
Console.WriteLine($"Found {totalLogGroups:N0} log groups. Deleted {deletedLogGroups:N0}. Skipped {skippedLogGroups:N0}.");
}
