private static async Task <List <FunctionConfiguration> > GetLambdaVersions(FunctionVersion functionVersion, AWSEnvironment environment) { var versionNumber = functionVersion != null ? Constants.LambdaAll : Constants.LambdaLastVersion; Console.WriteLine($"Reading lambda function versions: {versionNumber}"); var result = new List <FunctionConfiguration>(); var awsConfiguration = new AmazonLambdaConfig() { RegionEndpoint = RegionEndpoint.GetBySystemName(environment.Region) }; var awsCredentials = new BasicAWSCredentials(environment.AccessKey, environment.SecretKey); string marker = null; using (var awsClient = new AmazonLambdaClient(awsCredentials, awsConfiguration)) { do { var response = await awsClient.ListFunctionsAsync(new ListFunctionsRequest { Marker = marker, FunctionVersion = functionVersion }); //marker =Task<List<ListFunctionsRequest>> response.Result.NextMarker; marker = response.NextMarker; result.AddRange(response.Functions.Where(x => x.FunctionName.StartsWith($"{environment.Name.ToString()}"))); } while (!string.IsNullOrEmpty(marker)); return(result); } }
public async Task <ListFunctionsResponse> GetFunctions(ListFunctionsRequest request) { using (var amazonLambdaClient = new AmazonLambdaClient(awsCredentials, RegionEndpoint.GetBySystemName(Region))) { var response = await amazonLambdaClient.ListFunctionsAsync(request); return(response); } }
public static async Task <List <FunctionConfiguration> > GetAllFunctions() { List <FunctionConfiguration> _functionList = new List <FunctionConfiguration>(); ListFunctionsResponse _response = new ListFunctionsResponse(); do { _response = (await _lambdaClient.ListFunctionsAsync(new ListFunctionsRequest { Marker = _response.NextMarker ?? null })); _functionList.AddRange(_response.Functions); }while (_response.NextMarker != null); return(_functionList); }
public async Task ListLambdasAsync(string awsProfile, string awsRegion) { Console.WriteLine(); // initialize AWS profile await InitializeAwsProfile(awsProfile, awsRegion : awsRegion); var cfnClient = new AmazonCloudFormationClient(AWSConfigs.RegionEndpoint); var lambdaClient = new AmazonLambdaClient(AWSConfigs.RegionEndpoint); var logsClient = new AmazonCloudWatchLogsClient(AWSConfigs.RegionEndpoint); // fetch all Lambda functions on account var globalFunctions = (await ListLambdasAsync()) .ToDictionary(function => function.FunctionName, function => function); // fetch all stacks on account var stacks = await ListStacksAsync(); Console.WriteLine($"Analyzing {stacks.Count():N0} CloudFormation stacks and {globalFunctions.Count():N0} Lambda functions"); // fetch most recent CloudWatch log stream for each Lambda function var logStreamsTask = Task.Run(async() => (await Task.WhenAll(globalFunctions.Select(async kv => { try { var response = await logsClient.DescribeLogStreamsAsync(new DescribeLogStreamsRequest { Descending = true, LogGroupName = $"/aws/lambda/{kv.Value.FunctionName}", OrderBy = OrderBy.LastEventTime, Limit = 1 }); return(Name: kv.Value.FunctionName, Streams: response.LogStreams.FirstOrDefault()); } catch { // log group doesn't exist return(Name: kv.Value.FunctionName, Streams: null); } }))).ToDictionary(tuple => tuple.Name, tuple => tuple.Streams)); // fetch all functions belonging to a CloudFormation stack var stacksWithFunctionsTask = Task.Run(async() => stacks.Zip( await Task.WhenAll(stacks.Select(stack => ListStackFunctionsAsync(stack.StackId))), (stack, stackFunctions) => (Stack: stack, Functions: stackFunctions) ).ToList() ); // wait for both fetch operations to finish await Task.WhenAll(logStreamsTask, stacksWithFunctionsTask); var logStreams = logStreamsTask.GetAwaiter().GetResult(); var stacksWithFunctions = stacksWithFunctionsTask.GetAwaiter().GetResult(); // remove all the functions that were discovered inside a stack from the orphaned list of functions foreach (var function in stacksWithFunctions.SelectMany(stackWithFunctions => stackWithFunctions.Functions)) { globalFunctions.Remove(function.Configuration.FunctionName); } // compute the max width for the function name (use logical ID if it belongs to the stack) var maxFunctionNameWidth = stacksWithFunctions .SelectMany(stackWithFunctions => stackWithFunctions.Functions) .Select(function => function.Name.Length) .Union(globalFunctions.Values.Select(function => function.FunctionName.Length)) .Append(0) .Max(); // compute max width for the function runtime name var maxRuntimeWidth = stacksWithFunctions .SelectMany(stackWithFunctions => stackWithFunctions.Functions) .Select(stackFunction => stackFunction.Configuration.Runtime.ToString().Length) .Union(globalFunctions.Values.Select(function => function.Runtime.ToString().Length)) .Append(0) .Max(); // print Lambda functions belonging to stacks var showAsteriskExplanation = false; foreach (var stackWithFunctions in stacksWithFunctions .Where(stackWithFunction => stackWithFunction.Functions.Any()) .OrderBy(stackWithFunction => stackWithFunction.Stack.StackName) ) { Console.WriteLine(); // check if CloudFormation stack was deployed by LambdaSharp var moduleInfoOutput = stackWithFunctions.Stack.Outputs .FirstOrDefault(output => (output.OutputKey == "ModuleInfo") || (output.OutputKey == "Module")) ?.OutputValue; var lambdaSharpToolOutput = stackWithFunctions.Stack.Outputs .FirstOrDefault(output => output.OutputKey == "LambdaSharpTool") ?.OutputValue; // NOTE (2020-05-06, bjorg): pre-0.6, the module information was emitted as two output values var moduleNameOutput = stackWithFunctions.Stack.Outputs .FirstOrDefault(output => output.OutputKey == "ModuleName") ?.OutputValue; var moduleVersionOutput = stackWithFunctions.Stack.Outputs .FirstOrDefault(output => output.OutputKey == "ModuleVersion") ?.OutputValue; // show CloudFormation stack name and optionally LambdaSharp module information Console.Write($"{Settings.OutputColor}{stackWithFunctions.Stack.StackName}{Settings.ResetColor}"); if (ModuleInfo.TryParse(moduleInfoOutput, out var moduleInfo)) { Console.Write($" ({Settings.InfoColor}{moduleInfo.FullName}:{moduleInfo.Version}{Settings.ResetColor}) [lash {lambdaSharpToolOutput ?? "pre-0.6.1"}]"); } else if ((moduleNameOutput != null) && (moduleVersionOutput != null)) { Console.Write($" ({Settings.InfoColor}{moduleNameOutput}:{moduleVersionOutput}{Settings.ResetColor}) [lash pre-0.6]"); } Console.WriteLine(":"); foreach (var function in stackWithFunctions.Functions.OrderBy(function => function.Name)) { PrintFunction(function.Name, function.Configuration); } } // print orphan Lambda functions if (globalFunctions.Any()) { Console.WriteLine(); Console.WriteLine("ORPHANS:"); foreach (var function in globalFunctions.Values.OrderBy(function => function.FunctionName)) { PrintFunction(function.FunctionName, function); } } // show optional (*) explanation if it was printed if (showAsteriskExplanation) { Console.WriteLine(); Console.WriteLine("(*) Showing Lambda last-modified date, because last event timestamp in CloudWatch log stream is not available"); } // local functions async Task <IEnumerable <FunctionConfiguration> > ListLambdasAsync() { var result = new List <FunctionConfiguration>(); var request = new ListFunctionsRequest(); do { var response = await lambdaClient.ListFunctionsAsync(request); result.AddRange(response.Functions); request.Marker = response.NextMarker; } while(request.Marker != null); return(result); } async Task <IEnumerable <Stack> > ListStacksAsync() { var result = new List <Stack>(); var request = new DescribeStacksRequest(); do { var response = await cfnClient.DescribeStacksAsync(request); result.AddRange(response.Stacks); request.NextToken = response.NextToken; } while(request.NextToken != null); return(result); } async Task <IEnumerable <(string Name, FunctionConfiguration Configuration)> > ListStackFunctionsAsync(string stackName) { var result = new List <(string, FunctionConfiguration)>(); var request = new ListStackResourcesRequest { StackName = stackName }; do { var attempts = 0; again: try { var response = await cfnClient.ListStackResourcesAsync(request); result.AddRange( response.StackResourceSummaries .Where(resourceSummary => resourceSummary.ResourceType == "AWS::Lambda::Function") .Select(summary => { globalFunctions.TryGetValue(summary.PhysicalResourceId, out var configuration); return(Name: summary.LogicalResourceId, Configuration: configuration); }) .Where(tuple => tuple.Configuration != null) ); request.NextToken = response.NextToken; } catch (AmazonCloudFormationException e) when( (e.Message == "Rate exceeded") && (++attempts < 30) ) { await Task.Delay(TimeSpan.FromSeconds(attempts)); goto again; } } while(request.NextToken != null); return(result); } void PrintFunction(string name, FunctionConfiguration function) { Console.Write(" "); Console.Write(name); Console.Write("".PadRight(maxFunctionNameWidth - name.Length + 4)); Console.Write(function.Runtime); Console.Write("".PadRight(maxRuntimeWidth - function.Runtime.ToString().Length + 4)); if ( !logStreams.TryGetValue(function.FunctionName, out var logStream) || (logStream?.LastEventTimestamp == null) ) { Console.Write(DateTimeOffset.Parse(function.LastModified).ToString("yyyy-MM-dd")); Console.Write("(*)"); showAsteriskExplanation = true; } else { Console.Write(logStream.LastEventTimestamp.ToString("yyyy-MM-dd")); } Console.WriteLine(); } }
public async Task DeleteOrphanLogsAsync(bool dryRun, string awsProfile, string awsRegion) { Console.WriteLine(); // initialize AWS profile await InitializeAwsProfile(awsProfile, awsRegion : awsRegion); var logsClient = new AmazonCloudWatchLogsClient(AWSConfigs.RegionEndpoint); // delete orphaned logs var totalLogGroups = 0; var activeLogGroups = 0; var orphanedLogGroups = 0; var skippedLogGroups = 0; await DeleteOrphanLambdaLogsAsync(); await DeleteOrphanApiGatewayLogs(); await DeleteOrphanApiGatewayV2Logs(); if ((orphanedLogGroups > 0) || (skippedLogGroups > 0)) { Console.WriteLine(); } Console.WriteLine($"Found {totalLogGroups:N0} log groups. Active {activeLogGroups:N0}. Orphaned {orphanedLogGroups:N0}. Skipped {skippedLogGroups:N0}."); // local functions async Task DeleteOrphanLambdaLogsAsync() { // list all lambda functions var lambdaClient = new AmazonLambdaClient(AWSConfigs.RegionEndpoint); var request = new ListFunctionsRequest { }; var lambdaLogGroupNames = new HashSet <string>(); do { var response = await lambdaClient.ListFunctionsAsync(request); foreach (var function in response.Functions) { lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}"); } request.Marker = response.NextMarker; } while(request.Marker != null); // list all log groups for lambda functions await DeleteOrphanCloudWatchLogs( "/aws/lambda/", logGroupName => lambdaLogGroupNames.Contains(logGroupName), logGroupName => Regex.IsMatch(logGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$") ); } async Task DeleteOrphanApiGatewayLogs() { // list all API Gateway V1 instances var apiGatewayClient = new AmazonAPIGatewayClient(AWSConfigs.RegionEndpoint); var request = new GetRestApisRequest { }; var apiGatewayGroupNames = new List <string>(); do { var response = await apiGatewayClient.GetRestApisAsync(request); apiGatewayGroupNames.AddRange(response.Items.Select(item => $"API-Gateway-Execution-Logs_{item.Id}/")); request.Position = response.Position; } while(request.Position != null); // list all log groups for API Gateway instances await DeleteOrphanCloudWatchLogs( "API-Gateway-Execution-Logs_", logGroupName => apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)), logGroupName => Regex.IsMatch(logGroupName, @"^API-Gateway-Execution-Logs_[a-zA-Z0-9]+/.+$") ); } async Task DeleteOrphanApiGatewayV2Logs() { // list all API Gateway V2 instances var apiGatewayV2Client = new AmazonApiGatewayV2Client(AWSConfigs.RegionEndpoint); var request = new GetApisRequest { }; var apiGatewayGroupNames = new List <string>(); do { var response = await apiGatewayV2Client.GetApisAsync(request); apiGatewayGroupNames.AddRange(response.Items.Select(item => $"/aws/apigateway/{item.ApiId}/")); request.NextToken = response.NextToken; } while(request.NextToken != null); // list all log groups for API Gateway instances await DeleteOrphanCloudWatchLogs( "/aws/apigateway/", logGroupName => (logGroupName == "/aws/apigateway/welcome") || apiGatewayGroupNames.Any(apiGatewayGroupName => logGroupName.StartsWith(apiGatewayGroupName, StringComparison.Ordinal)), logGroupName => Regex.IsMatch(logGroupName, @"^/aws/apigateway/[a-zA-Z0-9]+/.+$") ); } async Task DeleteOrphanCloudWatchLogs(string logGroupPrefix, Func <string, bool> isActiveLogGroup, Func <string, bool> isValidLogGroup) { var describeLogGroupsRequest = new DescribeLogGroupsRequest { LogGroupNamePrefix = logGroupPrefix }; do { var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest); totalLogGroups += describeLogGroupsResponse.LogGroups.Count; foreach (var logGroup in describeLogGroupsResponse.LogGroups) { if (isActiveLogGroup(logGroup.LogGroupName)) { // nothing to do ++activeLogGroups; } else if (isValidLogGroup(logGroup.LogGroupName)) { // attempt to delete log group if (dryRun) { Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)"); ++orphanedLogGroups; } else { try { await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest { LogGroupName = logGroup.LogGroupName }); Console.WriteLine($"* deleted '{logGroup.LogGroupName}'"); ++orphanedLogGroups; } catch { LogError($"could not delete '{logGroup.LogGroupName}'"); ++skippedLogGroups; } } } else { // log group has an invalid name structure; skip it Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'"); ++skippedLogGroups; } } describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken; } while(describeLogGroupsRequest.NextToken != null); } }
public async Task <List <string> > GetFunctionList() { var response = await client.ListFunctionsAsync(); return(null); }
public async Task DeleteOrphanLambdaLogsAsync(bool dryRun) { Console.WriteLine(); // list all lambda functions var lambdaClient = new AmazonLambdaClient(); var listFunctionsRequest = new ListFunctionsRequest { }; var lambdaLogGroupNames = new HashSet <string>(); do { var listFunctionsResponse = await lambdaClient.ListFunctionsAsync(listFunctionsRequest); foreach (var function in listFunctionsResponse.Functions) { lambdaLogGroupNames.Add($"/aws/lambda/{function.FunctionName}"); } listFunctionsRequest.Marker = listFunctionsResponse.NextMarker; } while(listFunctionsRequest.Marker != null); // list all log groups for lambda functions var logsClient = new AmazonCloudWatchLogsClient(); var describeLogGroupsRequest = new DescribeLogGroupsRequest { LogGroupNamePrefix = "/aws/lambda/" }; var totalLogGroups = 0; var deletedLogGroups = 0; var skippedLogGroups = 0; do { var describeLogGroupsResponse = await logsClient.DescribeLogGroupsAsync(describeLogGroupsRequest); totalLogGroups += describeLogGroupsResponse.LogGroups.Count; foreach (var logGroup in describeLogGroupsResponse.LogGroups) { if (lambdaLogGroupNames.Contains(logGroup.LogGroupName)) { // nothing to do } else if (System.Text.RegularExpressions.Regex.IsMatch(logGroup.LogGroupName, @"^\/aws\/lambda\/[a-zA-Z0-9\-_]+$")) { // attempt to delete log group if (dryRun) { Console.WriteLine($"* deleted '{logGroup.LogGroupName}' (skipped)"); } else { try { await logsClient.DeleteLogGroupAsync(new DeleteLogGroupRequest { LogGroupName = logGroup.LogGroupName }); Console.WriteLine($"* deleted '{logGroup.LogGroupName}'"); ++deletedLogGroups; } catch { LogError($"could not delete '{logGroup.LogGroupName}'"); } } } else { // log group has an invalid name structure; skip it Console.WriteLine($"SKIPPED '{logGroup.LogGroupName}'"); ++skippedLogGroups; } } describeLogGroupsRequest.NextToken = describeLogGroupsResponse.NextToken; } while(describeLogGroupsRequest.NextToken != null); if ((deletedLogGroups > 0) || (skippedLogGroups > 0)) { Console.WriteLine(); } Console.WriteLine($"Found {totalLogGroups:N0} log groups. Deleted {deletedLogGroups:N0}. Skipped {skippedLogGroups:N0}."); }