internal DescribeLogGroupsResponse DescribeLogGroups(DescribeLogGroupsRequest request)
{
var marshaller = new DescribeLogGroupsRequestMarshaller();
var unmarshaller = DescribeLogGroupsResponseUnmarshaller.Instance;
return(Invoke <DescribeLogGroupsRequest, DescribeLogGroupsResponse>(request, marshaller, unmarshaller));
}
// snippet-start:[CloudWatchLogs.dotnetv3.DescribeLogGroupsExample]
public static async Task Main()
{
// Creates a CloudWatch Logs client using the default
// user. If you need to work with resources in another
// AWS Region than the one defined for the default user,
// pass the AWS Region as a parameter to the client constructor.
var client = new AmazonCloudWatchLogsClient();
var request = new DescribeLogGroupsRequest
{
Limit = 5,
};
var response = await client.DescribeLogGroupsAsync(request);
if (response.LogGroups.Count > 0)
{
do
{
response.LogGroups.ForEach(lg =>
{
Console.WriteLine($"{lg.LogGroupName} is associated with the key: {lg.KmsKeyId}.");
Console.WriteLine($"Created on: {lg.CreationTime.Date.Date}");
Console.WriteLine($"Date for this group will be stored for: {lg.RetentionInDays} days.\n");
});
}while (response.NextToken is not null);
}
}
/// <summary>
/// Creates the log group.
/// </summary>
/// <exception cref="Serilog.Sinks.AwsCloudWatch.AwsCloudWatchSinkException"></exception>
private async Task CreateLogGroupAsync()
{
if (options.CreateLogGroup)
{
// see if the log group already exists
var describeRequest = new DescribeLogGroupsRequest
{
LogGroupNamePrefix = options.LogGroupName
};
var logGroups = await cloudWatchClient
.DescribeLogGroupsAsync(describeRequest);
var logGroup = logGroups
.LogGroups
.FirstOrDefault(lg => string.Equals(lg.LogGroupName, options.LogGroupName, StringComparison.Ordinal));
// create log group if it doesn't exist
if (logGroup == null)
{
var createRequest = new CreateLogGroupRequest(options.LogGroupName);
var createResponse = await cloudWatchClient.CreateLogGroupAsync(createRequest);
// update the retention policy if a specific period is defined
if (options.LogGroupRetentionPolicy != LogGroupRetentionPolicy.Indefinitely)
{
var putRetentionRequest = new PutRetentionPolicyRequest(options.LogGroupName, (int)options.LogGroupRetentionPolicy);
await cloudWatchClient.PutRetentionPolicyAsync(putRetentionRequest);
}
}
}
}
/// <summary>
/// Initiates the asynchronous execution of the DescribeLogGroups operation.
/// <seealso cref="Amazon.CloudWatchLogs.IAmazonCloudWatchLogs"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the DescribeLogGroups operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task <DescribeLogGroupsResponse> DescribeLogGroupsAsync(DescribeLogGroupsRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new DescribeLogGroupsRequestMarshaller();
var unmarshaller = DescribeLogGroupsResponseUnmarshaller.Instance;
return(Invoke <IRequest, DescribeLogGroupsRequest, DescribeLogGroupsResponse>(request, marshaller, unmarshaller, signer, cancellationToken));
}
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonCloudWatchLogsConfig config = new AmazonCloudWatchLogsConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonCloudWatchLogsClient client = new AmazonCloudWatchLogsClient(creds, config);
DescribeLogGroupsResponse resp = new DescribeLogGroupsResponse();
do
{
DescribeLogGroupsRequest req = new DescribeLogGroupsRequest
{
NextToken = resp.NextToken
,
Limit = maxItems
};
resp = client.DescribeLogGroups(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.LogGroups)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.NextToken));
}
public async Task DescribeLogGroupsTest_TwoResponses()
{
var request = new DescribeLogGroupsRequest();
var firstResponse = new DescribeLogGroupsResponse()
{
NextToken = "foo"
};
var secondResponse = new DescribeLogGroupsResponse()
{
NextToken = null
};
var token = new CancellationToken();
_mockClient.SetupSequence(x => x.DescribeLogGroupsAsync(It.IsAny <DescribeLogGroupsRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(firstResponse)
.ReturnsAsync(secondResponse);
var numResponses = 0;
await foreach (var desination in _mockClient.Object.Paginators.DescribeLogGroups(request).Responses.WithCancellation(token))
{
numResponses += 1;
}
Assert.Equal(2, numResponses);
}
private async Task <IEnumerable <string> > GetLogGroups(IEnumerable <string> logGroups = null, string nextToken = default)
{
while (true)
{
if (logGroups == null)
{
logGroups = new string[0];
}
var request = new DescribeLogGroupsRequest {
Limit = 50, LogGroupNamePrefix = _prefix, NextToken = nextToken
};
var result = await _cloudWatchLogs.DescribeLogGroupsAsync(request);
logGroups = logGroups.Concat(result.LogGroups.Select(logGroup => logGroup.LogGroupName));
if (!string.IsNullOrEmpty(result.NextToken))
{
nextToken = result.NextToken;
}
else
{
return(logGroups);
}
}
}
/// <summary>
/// To check if the credentials is right
/// </summary>
/// <returns>Return a bool verb after trying to connect to AWS</returns>
public bool awsConnection()
{
awsLogGroup = new DescribeLogGroupsRequest();
try
{
awsCloudwatchClient.DescribeLogGroups(awsLogGroup);
return(true);
}
catch { return(false); }
}
public LogGroupsReader(InputContext context, IRowFactory rowFactory)
{
_context = context;
_rowFactory = rowFactory;
_client = new AmazonCloudWatchLogsClient();
CheckFieldTypes();
_request = new DescribeLogGroupsRequest()
{
Limit = GetLimit(), LogGroupNamePrefix = GetPrefix()
};
}
private static bool LogGroupExists(IAmazonCloudWatchLogs client, string logGroupName)
{
var describeLogGroupRequest = new DescribeLogGroupsRequest
{
LogGroupNamePrefix = logGroupName,
Limit = 1
};
var describeLogGroupResponse =
(client.DescribeLogGroupsAsync(describeLogGroupRequest)).GetAwaiter()
.GetResult();
return(describeLogGroupResponse.LogGroups.Any(x => x.LogGroupName == logGroupName));
}
internal DescribeLogGroupsResponse DescribeLogGroups(DescribeLogGroupsRequest request)
{
var task = DescribeLogGroupsAsync(request);
try
{
return(task.Result);
}
catch (AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return(null);
}
}
public async Task DescribeLogGroupsTest__OnlyUsedOnce()
{
var request = new DescribeLogGroupsRequest();
var response = new DescribeLogGroupsResponse();
var token = new CancellationToken();
var paginator = _mockClient.Object.Paginators.DescribeLogGroups(request);
_mockClient.Setup(x => x.DescribeLogGroupsAsync(It.IsAny <DescribeLogGroupsRequest>(), It.IsAny <CancellationToken>())).ReturnsAsync(response);
// Should work the first time
await LoopOverLogGroups(paginator, token);
// Second time should throw an exception
await Assert.ThrowsAsync <System.InvalidOperationException>(async() => await LoopOverLogGroups(paginator, token));
}
private async Task EnsureInitialized()
{
if (hasInit)
{
return;
}
if (options.CreateLogGroup)
{
// see if the log group already exists
DescribeLogGroupsRequest describeRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = options.LogGroupName
};
var logGroups = await cloudWatchClient.DescribeLogGroupsAsync(describeRequest);
var logGroup = logGroups.LogGroups.FirstOrDefault(lg => string.Equals(lg.LogGroupName, options.LogGroupName, StringComparison.OrdinalIgnoreCase));
// create log group if it doesn't exist
if (logGroup == null)
{
CreateLogGroupRequest createRequest = new CreateLogGroupRequest(options.LogGroupName);
var createResponse = await cloudWatchClient.CreateLogGroupAsync(createRequest);
if (!createResponse.HttpStatusCode.IsSuccessStatusCode())
{
throw new Exception($"Tried to create a log group, but failed with status code '{createResponse.HttpStatusCode}' and data '{createResponse.ResponseMetadata.FlattenedMetaData()}'.");
}
}
}
// create log stream
CreateLogStreamRequest createLogStreamRequest = new CreateLogStreamRequest()
{
LogGroupName = options.LogGroupName,
LogStreamName = logStreamName
};
var createLogStreamResponse = await cloudWatchClient.CreateLogStreamAsync(createLogStreamRequest);
if (!createLogStreamResponse.HttpStatusCode.IsSuccessStatusCode())
{
throw new Exception(
$"Tried to create a log stream, but failed with status code '{createLogStreamResponse.HttpStatusCode}' and data '{createLogStreamResponse.ResponseMetadata.FlattenedMetaData()}'.");
}
hasInit = true;
}
/// <summary>
/// Creates the log group.
/// </summary>
/// <exception cref="Serilog.Sinks.AwsCloudWatch.AwsCloudWatchSinkException"></exception>
private async Task CreateLogGroupAsync()
{
if (options.CreateLogGroup)
{
// see if the log group already exists
DescribeLogGroupsRequest describeRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = options.LogGroupName
};
var logGroups = await cloudWatchClient.DescribeLogGroupsAsync(describeRequest);
var logGroup = logGroups.LogGroups.FirstOrDefault(lg => string.Equals(lg.LogGroupName, options.LogGroupName, StringComparison.OrdinalIgnoreCase));
// create log group if it doesn't exist
if (logGroup == null)
{
CreateLogGroupRequest createRequest = new CreateLogGroupRequest(options.LogGroupName);
var createResponse = await cloudWatchClient.CreateLogGroupAsync(createRequest);
}
}
}
public async Task DescribeLogGroupsTest__CancellationToken()
{
var request = new DescribeLogGroupsRequest();
var firstResponse = new DescribeLogGroupsResponse()
{
NextToken = "foo"
};
var secondResponse = new DescribeLogGroupsResponse()
{
NextToken = null
};
var paginator = _mockClient.Object.Paginators.DescribeLogGroups(request);
var tokenSource = new CancellationTokenSource();
var token = tokenSource.Token;
tokenSource.Cancel();
_mockClient.SetupSequence(x => x.DescribeLogGroupsAsync(It.IsAny <DescribeLogGroupsRequest>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(firstResponse)
.ReturnsAsync(secondResponse);
await Assert.ThrowsAsync <OperationCanceledException>(async() => await LoopOverLogGroups(paginator, token));
}
/// <summary>
/// Get all the LogGroup on the RegionEndpoint
/// </summary>
/// <returns>return the LogGroup found</returns>
public DescribeLogGroupsResponse GetLogGroups()
{
awsLogGroup = new DescribeLogGroupsRequest();
return(awsCloudwatchClient.DescribeLogGroups(awsLogGroup));
}
public async Task DeleteOrphanLogsAsync(bool dryRun, string awsProfile, string awsRegion)
{
Console.WriteLine();
// initialize AWS profile
await InitializeAwsProfile(awsProfile, awsRegion : awsRegion);
var logsClient = new AmazonCloudWatchLogsClient(AWSConfigs.RegionEndpoint);
// delete orphaned logs
var totalLogGroups = 0;
var activeLogGroups = 0;
var orphanedLogGroups = 0;
var skippedLogGroups = 0;
await DeleteOrphanLambdaLogsAsync();
await DeleteOrphanApiGatewayLogs();
await DeleteOrphanApiGatewayV2Logs();
if ((orphanedLogGroups > 0) || (skippedLogGroups > 0))
{
Console.WriteLine();
}
Console.WriteLine($"Found {totalLogGroups:N0} log groups. Active {activeLogGroups:N0}. Orphaned {orphanedLogGroups:N0}. Skipped {skippedLogGroups:N0}.");
// local functions
async Task DeleteOrphanLambdaLogsAsync()
{
// list all lambda functions
var lambdaClient = new AmazonLambdaClient(AWSConfigs.RegionEndpoint);
var request = new ListFunctionsRequest {
};
var lambdaLogGroupNames = new HashSet <string>();
do
{
var response = await lambdaClient.ListFunctionsAsync(request);
foreach (var function in response.Functions)
{
lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}");
}
request.Marker = response.NextMarker;
} while(request.Marker != null);
// list all log groups for lambda functions
await DeleteOrphanCloudWatchLogs(
"/aws/lambda/",
logGroupName => lambdaLogGroupNames.Contains(logGroupName),
logGroupName => Regex.IsMatch(logGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$")
);
}
async Task DeleteOrphanApiGatewayLogs()
{
// list all API Gateway V1 instances
var apiGatewayClient = new AmazonAPIGatewayClient(AWSConfigs.RegionEndpoint);
var request = new GetRestApisRequest {
};
var apiGatewayGroupNames = new List <string>();
do
{
var response = await apiGatewayClient.GetRestApisAsync(request);
apiGatewayGroupNames.AddRange(response.Items.Select(item => $"API-Gateway-Execution-Logs_{item.Id}/"));
request.Position = response.Position;
} while(request.Position != null);
// list all log groups for API Gateway instances
await DeleteOrphanCloudWatchLogs(
"API-Gateway-Execution-Logs_",
logGroupName => apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)),
logGroupName => Regex.IsMatch(logGroupName, @"^API-Gateway-Execution-Logs_[a-zA-Z0-9]+/.+$")
);
}
async Task DeleteOrphanApiGatewayV2Logs()
{
// list all API Gateway V2 instances
var apiGatewayV2Client = new AmazonApiGatewayV2Client(AWSConfigs.RegionEndpoint);
var request = new GetApisRequest {
};
var apiGatewayGroupNames = new List <string>();
do
{
var response = await apiGatewayV2Client.GetApisAsync(request);
apiGatewayGroupNames.AddRange(response.Items.Select(item => $"/aws/apigateway/{item.ApiId}/"));
request.NextToken = response.NextToken;
} while(request.NextToken != null);
// list all log groups for API Gateway instances
await DeleteOrphanCloudWatchLogs(
"/aws/apigateway/",
logGroupName => (logGroupName == "/aws/apigateway/welcome") || apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)),
logGroupName => Regex.IsMatch(logGroupName, @"^/aws/apigateway/[a-zA-Z0-9]+/.+$")
);
}
async Task DeleteOrphanCloudWatchLogs(string logGroupPrefix, Func <string, bool> isActiveLogGroup, Func <string, bool> isValidLogGroup)
{
var describeLogGroupsRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = logGroupPrefix
};
do
{
var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest);
totalLogGroups += describeLogGroupsResponse.LogGroups.Count;
foreach (var logGroup in describeLogGroupsResponse.LogGroups)
{
if (isActiveLogGroup(logGroup.LogGroupName))
{
// nothing to do
++activeLogGroups;
}
else if (isValidLogGroup(logGroup.LogGroupName))
{
// attempt to delete log group
if (dryRun)
{
Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)");
++orphanedLogGroups;
}
else
{
try {
await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest {
LogGroupName = logGroup.LogGroupName
});
Console.WriteLine($"* deleted '{logGroup.LogGroupName}'");
++orphanedLogGroups;
} catch {
LogError($"could not delete '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
}
else
{
// log group has an invalid name structure; skip it
Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken;
} while(describeLogGroupsRequest.NextToken != null);
}
}
public async Task DeleteOrphanLambdaLogsAsync(bool dryRun)
{
Console.WriteLine();
// list all lambda functions
var lambdaClient = new AmazonLambdaClient();
var listFunctionsRequest = new ListFunctionsRequest {
};
var lambdaLogGroupNames = new HashSet <string>();
do
{
var listFunctionsResponse = await lambdaClient.ListFunctionsAsync(listFunctionsRequest);
foreach (var function in listFunctionsResponse.Functions)
{
lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}");
}
listFunctionsRequest.Marker = listFunctionsResponse.NextMarker;
} while(listFunctionsRequest.Marker != null);
// list all log groups for lambda functions
var logsClient = new AmazonCloudWatchLogsClient();
var describeLogGroupsRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = "/aws/lambda/"
};
var totalLogGroups = 0;
var deletedLogGroups = 0;
var skippedLogGroups = 0;
do
{
var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest);
totalLogGroups += describeLogGroupsResponse.LogGroups.Count;
foreach (var logGroup in describeLogGroupsResponse.LogGroups)
{
if (lambdaLogGroupNames.Contains(logGroup.LogGroupName))
{
// nothing to do
}
else if (System.Text.RegularExpressions.Regex.IsMatch(logGroup.LogGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$"))
{
// attempt to delete log group
if (dryRun)
{
Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)");
}
else
{
try {
await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest {
LogGroupName = logGroup.LogGroupName
});
Console.WriteLine($"* deleted '{logGroup.LogGroupName}'");
++deletedLogGroups;
} catch {
LogError($"could not delete '{logGroup.LogGroupName}'");
}
}
}
else
{
// log group has an invalid name structure; skip it
Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken;
} while(describeLogGroupsRequest.NextToken != null);
if ((deletedLogGroups > 0) || (skippedLogGroups > 0))
{
Console.WriteLine();
}
Console.WriteLine($"Found {totalLogGroups:N0} log groups. Deleted {deletedLogGroups:N0}. Skipped {skippedLogGroups:N0}.");
}
