private static async Task <List <FunctionConfiguration> > GetLambdaVersions(FunctionVersion functionVersion, AWSEnvironment environment)
{
var versionNumber = functionVersion != null ? Constants.LambdaAll : Constants.LambdaLastVersion;
Console.WriteLine($"Reading lambda function versions: {versionNumber}");
var result = new List <FunctionConfiguration>();
var awsConfiguration = new AmazonLambdaConfig()
{
RegionEndpoint = RegionEndpoint.GetBySystemName(environment.Region)
};
var awsCredentials = new BasicAWSCredentials(environment.AccessKey, environment.SecretKey);
string marker = null;
using (var awsClient = new AmazonLambdaClient(awsCredentials, awsConfiguration))
{
do
{
var response = await awsClient.ListFunctionsAsync(new ListFunctionsRequest
{
Marker = marker,
FunctionVersion = functionVersion
});
//marker =Task<List<ListFunctionsRequest>> response.Result.NextMarker;
marker = response.NextMarker;
result.AddRange(response.Functions.Where(x => x.FunctionName.StartsWith($"{environment.Name.ToString()}")));
} while (!string.IsNullOrEmpty(marker));
return(result);
}
}
public async Task <ListFunctionsResponse> GetFunctions(ListFunctionsRequest request)
{
using (var amazonLambdaClient = new AmazonLambdaClient(awsCredentials, RegionEndpoint.GetBySystemName(Region)))
{
var response = await amazonLambdaClient.ListFunctionsAsync(request);
return(response);
}
}
public static async Task <List <FunctionConfiguration> > GetAllFunctions()
{
List <FunctionConfiguration> _functionList = new List <FunctionConfiguration>();
ListFunctionsResponse _response = new ListFunctionsResponse();
do
{
_response = (await _lambdaClient.ListFunctionsAsync(new ListFunctionsRequest {
Marker = _response.NextMarker ?? null
}));
_functionList.AddRange(_response.Functions);
}while (_response.NextMarker != null);
return(_functionList);
}
public async Task ListLambdasAsync(string awsProfile, string awsRegion)
{
Console.WriteLine();
// initialize AWS profile
await InitializeAwsProfile(awsProfile, awsRegion : awsRegion);
var cfnClient = new AmazonCloudFormationClient(AWSConfigs.RegionEndpoint);
var lambdaClient = new AmazonLambdaClient(AWSConfigs.RegionEndpoint);
var logsClient = new AmazonCloudWatchLogsClient(AWSConfigs.RegionEndpoint);
// fetch all Lambda functions on account
var globalFunctions = (await ListLambdasAsync())
.ToDictionary(function => function.FunctionName, function => function);
// fetch all stacks on account
var stacks = await ListStacksAsync();
Console.WriteLine($"Analyzing {stacks.Count():N0} CloudFormation stacks and {globalFunctions.Count():N0} Lambda functions");
// fetch most recent CloudWatch log stream for each Lambda function
var logStreamsTask = Task.Run(async() => (await Task.WhenAll(globalFunctions.Select(async kv => {
try {
var response = await logsClient.DescribeLogStreamsAsync(new DescribeLogStreamsRequest {
Descending = true,
LogGroupName = $"/aws/lambda/{kv.Value.FunctionName}",
OrderBy = OrderBy.LastEventTime,
Limit = 1
});
return(Name: kv.Value.FunctionName, Streams: response.LogStreams.FirstOrDefault());
} catch {
// log group doesn't exist
return(Name: kv.Value.FunctionName, Streams: null);
}
}))).ToDictionary(tuple => tuple.Name, tuple => tuple.Streams));
// fetch all functions belonging to a CloudFormation stack
var stacksWithFunctionsTask = Task.Run(async() => stacks.Zip(
await Task.WhenAll(stacks.Select(stack => ListStackFunctionsAsync(stack.StackId))),
(stack, stackFunctions) => (Stack: stack, Functions: stackFunctions)
).ToList()
);
// wait for both fetch operations to finish
await Task.WhenAll(logStreamsTask, stacksWithFunctionsTask);
var logStreams = logStreamsTask.GetAwaiter().GetResult();
var stacksWithFunctions = stacksWithFunctionsTask.GetAwaiter().GetResult();
// remove all the functions that were discovered inside a stack from the orphaned list of functions
foreach (var function in stacksWithFunctions.SelectMany(stackWithFunctions => stackWithFunctions.Functions))
{
globalFunctions.Remove(function.Configuration.FunctionName);
}
// compute the max width for the function name (use logical ID if it belongs to the stack)
var maxFunctionNameWidth = stacksWithFunctions
.SelectMany(stackWithFunctions => stackWithFunctions.Functions)
.Select(function => function.Name.Length)
.Union(globalFunctions.Values.Select(function => function.FunctionName.Length))
.Append(0)
.Max();
// compute max width for the function runtime name
var maxRuntimeWidth = stacksWithFunctions
.SelectMany(stackWithFunctions => stackWithFunctions.Functions)
.Select(stackFunction => stackFunction.Configuration.Runtime.ToString().Length)
.Union(globalFunctions.Values.Select(function => function.Runtime.ToString().Length))
.Append(0)
.Max();
// print Lambda functions belonging to stacks
var showAsteriskExplanation = false;
foreach (var stackWithFunctions in stacksWithFunctions
.Where(stackWithFunction => stackWithFunction.Functions.Any())
.OrderBy(stackWithFunction => stackWithFunction.Stack.StackName)
)
{
Console.WriteLine();
// check if CloudFormation stack was deployed by LambdaSharp
var moduleInfoOutput = stackWithFunctions.Stack.Outputs
.FirstOrDefault(output => (output.OutputKey == "ModuleInfo") || (output.OutputKey == "Module"))
?.OutputValue;
var lambdaSharpToolOutput = stackWithFunctions.Stack.Outputs
.FirstOrDefault(output => output.OutputKey == "LambdaSharpTool")
?.OutputValue;
// NOTE (2020-05-06, bjorg): pre-0.6, the module information was emitted as two output values
var moduleNameOutput = stackWithFunctions.Stack.Outputs
.FirstOrDefault(output => output.OutputKey == "ModuleName")
?.OutputValue;
var moduleVersionOutput = stackWithFunctions.Stack.Outputs
.FirstOrDefault(output => output.OutputKey == "ModuleVersion")
?.OutputValue;
// show CloudFormation stack name and optionally LambdaSharp module information
Console.Write($"{Settings.OutputColor}{stackWithFunctions.Stack.StackName}{Settings.ResetColor}");
if (ModuleInfo.TryParse(moduleInfoOutput, out var moduleInfo))
{
Console.Write($" ({Settings.InfoColor}{moduleInfo.FullName}:{moduleInfo.Version}{Settings.ResetColor}) [lash {lambdaSharpToolOutput ?? "pre-0.6.1"}]");
}
else if ((moduleNameOutput != null) && (moduleVersionOutput != null))
{
Console.Write($" ({Settings.InfoColor}{moduleNameOutput}:{moduleVersionOutput}{Settings.ResetColor}) [lash pre-0.6]");
}
Console.WriteLine(":");
foreach (var function in stackWithFunctions.Functions.OrderBy(function => function.Name))
{
PrintFunction(function.Name, function.Configuration);
}
}
// print orphan Lambda functions
if (globalFunctions.Any())
{
Console.WriteLine();
Console.WriteLine("ORPHANS:");
foreach (var function in globalFunctions.Values.OrderBy(function => function.FunctionName))
{
PrintFunction(function.FunctionName, function);
}
}
// show optional (*) explanation if it was printed
if (showAsteriskExplanation)
{
Console.WriteLine();
Console.WriteLine("(*) Showing Lambda last-modified date, because last event timestamp in CloudWatch log stream is not available");
}
// local functions
async Task <IEnumerable <FunctionConfiguration> > ListLambdasAsync()
{
var result = new List <FunctionConfiguration>();
var request = new ListFunctionsRequest();
do
{
var response = await lambdaClient.ListFunctionsAsync(request);
result.AddRange(response.Functions);
request.Marker = response.NextMarker;
} while(request.Marker != null);
return(result);
}
async Task <IEnumerable <Stack> > ListStacksAsync()
{
var result = new List <Stack>();
var request = new DescribeStacksRequest();
do
{
var response = await cfnClient.DescribeStacksAsync(request);
result.AddRange(response.Stacks);
request.NextToken = response.NextToken;
} while(request.NextToken != null);
return(result);
}
async Task <IEnumerable <(string Name, FunctionConfiguration Configuration)> > ListStackFunctionsAsync(string stackName)
{
var result = new List <(string, FunctionConfiguration)>();
var request = new ListStackResourcesRequest {
StackName = stackName
};
do
{
var attempts = 0;
again:
try {
var response = await cfnClient.ListStackResourcesAsync(request);
result.AddRange(
response.StackResourceSummaries
.Where(resourceSummary => resourceSummary.ResourceType == "AWS::Lambda::Function")
.Select(summary => {
globalFunctions.TryGetValue(summary.PhysicalResourceId, out var configuration);
return(Name: summary.LogicalResourceId, Configuration: configuration);
})
.Where(tuple => tuple.Configuration != null)
);
request.NextToken = response.NextToken;
} catch (AmazonCloudFormationException e) when(
(e.Message == "Rate exceeded") &&
(++attempts < 30)
)
{
await Task.Delay(TimeSpan.FromSeconds(attempts));
goto again;
}
} while(request.NextToken != null);
return(result);
}
void PrintFunction(string name, FunctionConfiguration function)
{
Console.Write(" ");
Console.Write(name);
Console.Write("".PadRight(maxFunctionNameWidth - name.Length + 4));
Console.Write(function.Runtime);
Console.Write("".PadRight(maxRuntimeWidth - function.Runtime.ToString().Length + 4));
if (
!logStreams.TryGetValue(function.FunctionName, out var logStream) ||
(logStream?.LastEventTimestamp == null)
)
{
Console.Write(DateTimeOffset.Parse(function.LastModified).ToString("yyyy-MM-dd"));
Console.Write("(*)");
showAsteriskExplanation = true;
}
else
{
Console.Write(logStream.LastEventTimestamp.ToString("yyyy-MM-dd"));
}
Console.WriteLine();
}
}
public async Task DeleteOrphanLogsAsync(bool dryRun, string awsProfile, string awsRegion)
{
Console.WriteLine();
// initialize AWS profile
await InitializeAwsProfile(awsProfile, awsRegion : awsRegion);
var logsClient = new AmazonCloudWatchLogsClient(AWSConfigs.RegionEndpoint);
// delete orphaned logs
var totalLogGroups = 0;
var activeLogGroups = 0;
var orphanedLogGroups = 0;
var skippedLogGroups = 0;
await DeleteOrphanLambdaLogsAsync();
await DeleteOrphanApiGatewayLogs();
await DeleteOrphanApiGatewayV2Logs();
if ((orphanedLogGroups > 0) || (skippedLogGroups > 0))
{
Console.WriteLine();
}
Console.WriteLine($"Found {totalLogGroups:N0} log groups. Active {activeLogGroups:N0}. Orphaned {orphanedLogGroups:N0}. Skipped {skippedLogGroups:N0}.");
// local functions
async Task DeleteOrphanLambdaLogsAsync()
{
// list all lambda functions
var lambdaClient = new AmazonLambdaClient(AWSConfigs.RegionEndpoint);
var request = new ListFunctionsRequest {
};
var lambdaLogGroupNames = new HashSet <string>();
do
{
var response = await lambdaClient.ListFunctionsAsync(request);
foreach (var function in response.Functions)
{
lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}");
}
request.Marker = response.NextMarker;
} while(request.Marker != null);
// list all log groups for lambda functions
await DeleteOrphanCloudWatchLogs(
"/aws/lambda/",
logGroupName => lambdaLogGroupNames.Contains(logGroupName),
logGroupName => Regex.IsMatch(logGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$")
);
}
async Task DeleteOrphanApiGatewayLogs()
{
// list all API Gateway V1 instances
var apiGatewayClient = new AmazonAPIGatewayClient(AWSConfigs.RegionEndpoint);
var request = new GetRestApisRequest {
};
var apiGatewayGroupNames = new List <string>();
do
{
var response = await apiGatewayClient.GetRestApisAsync(request);
apiGatewayGroupNames.AddRange(response.Items.Select(item => $"API-Gateway-Execution-Logs_{item.Id}/"));
request.Position = response.Position;
} while(request.Position != null);
// list all log groups for API Gateway instances
await DeleteOrphanCloudWatchLogs(
"API-Gateway-Execution-Logs_",
logGroupName => apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)),
logGroupName => Regex.IsMatch(logGroupName, @"^API-Gateway-Execution-Logs_[a-zA-Z0-9]+/.+$")
);
}
async Task DeleteOrphanApiGatewayV2Logs()
{
// list all API Gateway V2 instances
var apiGatewayV2Client = new AmazonApiGatewayV2Client(AWSConfigs.RegionEndpoint);
var request = new GetApisRequest {
};
var apiGatewayGroupNames = new List <string>();
do
{
var response = await apiGatewayV2Client.GetApisAsync(request);
apiGatewayGroupNames.AddRange(response.Items.Select(item => $"/aws/apigateway/{item.ApiId}/"));
request.NextToken = response.NextToken;
} while(request.NextToken != null);
// list all log groups for API Gateway instances
await DeleteOrphanCloudWatchLogs(
"/aws/apigateway/",
logGroupName => (logGroupName == "/aws/apigateway/welcome") || apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)),
logGroupName => Regex.IsMatch(logGroupName, @"^/aws/apigateway/[a-zA-Z0-9]+/.+$")
);
}
async Task DeleteOrphanCloudWatchLogs(string logGroupPrefix, Func <string, bool> isActiveLogGroup, Func <string, bool> isValidLogGroup)
{
var describeLogGroupsRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = logGroupPrefix
};
do
{
var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest);
totalLogGroups += describeLogGroupsResponse.LogGroups.Count;
foreach (var logGroup in describeLogGroupsResponse.LogGroups)
{
if (isActiveLogGroup(logGroup.LogGroupName))
{
// nothing to do
++activeLogGroups;
}
else if (isValidLogGroup(logGroup.LogGroupName))
{
// attempt to delete log group
if (dryRun)
{
Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)");
++orphanedLogGroups;
}
else
{
try {
await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest {
LogGroupName = logGroup.LogGroupName
});
Console.WriteLine($"* deleted '{logGroup.LogGroupName}'");
++orphanedLogGroups;
} catch {
LogError($"could not delete '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
}
else
{
// log group has an invalid name structure; skip it
Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken;
} while(describeLogGroupsRequest.NextToken != null);
}
}
public async Task <List <string> > GetFunctionList()
{
var response = await client.ListFunctionsAsync();
return(null);
}
public async Task DeleteOrphanLambdaLogsAsync(bool dryRun)
{
Console.WriteLine();
// list all lambda functions
var lambdaClient = new AmazonLambdaClient();
var listFunctionsRequest = new ListFunctionsRequest {
};
var lambdaLogGroupNames = new HashSet <string>();
do
{
var listFunctionsResponse = await lambdaClient.ListFunctionsAsync(listFunctionsRequest);
foreach (var function in listFunctionsResponse.Functions)
{
lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}");
}
listFunctionsRequest.Marker = listFunctionsResponse.NextMarker;
} while(listFunctionsRequest.Marker != null);
// list all log groups for lambda functions
var logsClient = new AmazonCloudWatchLogsClient();
var describeLogGroupsRequest = new DescribeLogGroupsRequest {
LogGroupNamePrefix = "/aws/lambda/"
};
var totalLogGroups = 0;
var deletedLogGroups = 0;
var skippedLogGroups = 0;
do
{
var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest);
totalLogGroups += describeLogGroupsResponse.LogGroups.Count;
foreach (var logGroup in describeLogGroupsResponse.LogGroups)
{
if (lambdaLogGroupNames.Contains(logGroup.LogGroupName))
{
// nothing to do
}
else if (System.Text.RegularExpressions.Regex.IsMatch(logGroup.LogGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$"))
{
// attempt to delete log group
if (dryRun)
{
Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)");
}
else
{
try {
await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest {
LogGroupName = logGroup.LogGroupName
});
Console.WriteLine($"* deleted '{logGroup.LogGroupName}'");
++deletedLogGroups;
} catch {
LogError($"could not delete '{logGroup.LogGroupName}'");
}
}
}
else
{
// log group has an invalid name structure; skip it
Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'");
++skippedLogGroups;
}
}
describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken;
} while(describeLogGroupsRequest.NextToken != null);
if ((deletedLogGroups > 0) || (skippedLogGroups > 0))
{
Console.WriteLine();
}
Console.WriteLine($"Found {totalLogGroups:N0} log groups. Deleted {deletedLogGroups:N0}. Skipped {skippedLogGroups:N0}.");
}