//--- Methods --- public async Task <bool> DeployChangeSetAsync( ModuleManifest manifest, ModuleNameMappings nameMappings, ModuleLocation moduleLocation, string stackName, bool allowDataLoss, bool protectStack, List <CloudFormationParameter> parameters ) { var now = DateTime.UtcNow; // check if cloudformation stack already exists and is in a final state Console.WriteLine(); Console.WriteLine($"Deploying stack: {stackName} [{moduleLocation.ModuleInfo}]"); var mostRecentStackEventId = await Settings.CfnClient.GetMostRecentStackEventIdAsync(stackName); // validate template (must have been copied to deployment bucket at this stage) if (moduleLocation.SourceBucketName != Settings.DeploymentBucketName) { LogError($"module source must match the deployment tier S3 bucket (EXPECTED: {Settings.DeploymentBucketName}, FOUND: {moduleLocation.SourceBucketName})"); return(false); } ValidateTemplateResponse validation; try { validation = await Settings.CfnClient.ValidateTemplateAsync(new ValidateTemplateRequest { TemplateURL = moduleLocation.ModuleTemplateUrl }); } catch (AmazonCloudFormationException e) { LogError($"{e.Message} (url: {moduleLocation.ModuleTemplateUrl})"); return(false); } // verify if we need to remove the old CloudFormation notification ARNs that were created by LambdaSharp config before v0.7 List <string> notificationARNs = null; ModuleNameMappings oldNameMappings = null; if (mostRecentStackEventId != null) { // fetch name mappings for current template; this is needed to properly map logical IDs to their original names when they get deleted oldNameMappings = await new ModelManifestLoader(Settings, "source").GetNameMappingsFromCloudFormationStackAsync(stackName); // NOTE (2019-09-19, bjorg): this is a HACK to remove the old notification ARNs, because doing it part // of the change set is not working for some reason. var deployedModule = await Settings.CfnClient.GetStackAsync(stackName, LogError); notificationARNs = deployedModule.Stack.NotificationARNs.Where(arn => !arn.Contains(":LambdaSharpTool-")).ToList(); if (notificationARNs.Count != deployedModule.Stack.NotificationARNs.Count) { Console.WriteLine("=> Removing legacy stack notification ARN"); var update = await Settings.CfnClient.UpdateStackAsync(new UpdateStackRequest { StackName = stackName, UsePreviousTemplate = true, Parameters = deployedModule.Stack.Parameters.Select(param => new CloudFormationParameter { ParameterKey = param.ParameterKey, UsePreviousValue = true }).ToList(), NotificationARNs = notificationARNs, Capabilities = validation.Capabilities }); var outcome = await Settings.CfnClient.TrackStackUpdateAsync(stackName, update.StackId, mostRecentStackEventId, nameMappings, oldNameMappings, LogError); if (!outcome.Success) { LogError("failed to remove legacy stack notification ARN; remove manually and try again"); return(false); } mostRecentStackEventId = await Settings.CfnClient.GetMostRecentStackEventIdAsync(stackName); Console.WriteLine("=> Legacy stack notification ARN has been removed"); } } // create change-set var success = false; var changeSetName = $"{moduleLocation.ModuleInfo.FullName.Replace(".", "-")}-{now:yyyy-MM-dd-hh-mm-ss}"; var updateOrCreate = (mostRecentStackEventId != null) ? "update" : "create"; var capabilities = validation.Capabilities.Any() ? "[" + string.Join(", ", validation.Capabilities) + "]" : ""; Console.WriteLine($"=> Stack {updateOrCreate} initiated for {Settings.InfoColor}{stackName}{Settings.ResetColor} {capabilities}"); CreateChangeSetResponse response; try { response = await Settings.CfnClient.CreateChangeSetAsync(new CreateChangeSetRequest { Capabilities = validation.Capabilities, ChangeSetName = changeSetName, ChangeSetType = (mostRecentStackEventId != null) ? ChangeSetType.UPDATE : ChangeSetType.CREATE, Description = $"Stack {updateOrCreate} {moduleLocation.ModuleInfo.FullName} (v{moduleLocation.ModuleInfo.Version})", Parameters = new List <CloudFormationParameter>(parameters) { new CloudFormationParameter { ParameterKey = "DeploymentPrefix", ParameterValue = Settings.TierPrefix }, new CloudFormationParameter { ParameterKey = "DeploymentPrefixLowercase", ParameterValue = Settings.TierPrefix.ToLowerInvariant() }, new CloudFormationParameter { ParameterKey = "DeploymentBucketName", ParameterValue = Settings.DeploymentBucketName }, new CloudFormationParameter { ParameterKey = "DeploymentChecksum", ParameterValue = manifest.TemplateChecksum } }, StackName = stackName, TemplateURL = moduleLocation.ModuleTemplateUrl, Tags = Settings.GetCloudFormationStackTags(moduleLocation.ModuleInfo.FullName, stackName) }); } catch (AmazonCloudFormationException e) { LogError($"cloudformation change-set failed: {e.Message}"); return(false); } try { var changes = await WaitForChangeSetAsync(response.Id); if (changes == null) { return(false); } if (!changes.Any()) { Console.WriteLine("=> No stack update required"); return(true); } // changes if (!allowDataLoss) { var lossyChanges = DetectLossyChanges(changes); if (lossyChanges.Any()) { Console.WriteLine(); Console.WriteLine($"{Settings.AlertColor}CAUTION:{Settings.ResetColor} detected potential replacement and data-loss in the following resources"); var maxResourceTypeWidth = lossyChanges.Select(change => change.ResourceChange.ResourceType.Length).Max(); foreach (var lossy in lossyChanges) { if (Settings.UseAnsiConsole) { if (lossy.ResourceChange.Replacement == Replacement.True) { Console.Write(AnsiTerminal.Red); Console.Write("ALWAYS "); } else { Console.Write(AnsiTerminal.Yellow); Console.Write("CONDITIONAL "); } Console.Write(AnsiTerminal.Reset); } else { Console.WriteLine((lossy.ResourceChange.Replacement == Replacement.True) ? "ALWAYS " : "CONDITIONAL " ); } Console.Write(lossy.ResourceChange.ResourceType); Console.Write("".PadRight(maxResourceTypeWidth - lossy.ResourceChange.ResourceType.Length + 4)); Console.Write(TranslateLogicalIdToFullName(lossy.ResourceChange.LogicalResourceId)); Console.WriteLine(); } if (!Settings.PromptYesNo("Proceed with potentially replacing/deleting resources?", false)) { return(false); } Console.WriteLine(); } } // execute change-set await Settings.CfnClient.ExecuteChangeSetAsync(new ExecuteChangeSetRequest { ChangeSetName = changeSetName, StackName = stackName }); var outcome = await Settings.CfnClient.TrackStackUpdateAsync( stackName, response.StackId, mostRecentStackEventId, nameMappings, oldNameMappings, LogError ); if (outcome.Success) { Console.WriteLine($"=> Stack {updateOrCreate} finished"); ShowStackResult(outcome.Stack); success = true; } else { Console.WriteLine($"=> Stack {updateOrCreate} FAILED"); } // optionally enable stack protection if (success) { // on success, protect the stack if requested if (protectStack) { await Settings.CfnClient.UpdateTerminationProtectionAsync(new UpdateTerminationProtectionRequest { EnableTerminationProtection = protectStack, StackName = stackName }); } } else if (mostRecentStackEventId == null) { // delete a new stack that failed to create try { await Settings.CfnClient.DeleteStackAsync(new DeleteStackRequest { StackName = stackName }); } catch { } } return(success); } finally { try { await Settings.CfnClient.DeleteChangeSetAsync(new DeleteChangeSetRequest { ChangeSetName = response.Id }); } catch { } } // local function string TranslateLogicalIdToFullName(string logicalId) { var fullName = logicalId; nameMappings?.ResourceNameMappings.TryGetValue(logicalId, out fullName); return(fullName ?? logicalId); } }
public static async Task <(Stack Stack, bool Success)> TrackStackUpdateAsync( this IAmazonCloudFormation cfnClient, string stackName, string stackId, string mostRecentStackEventId, ModuleNameMappings nameMappings = null, LogErrorDelegate logError = null ) { var seenEventIds = new HashSet <string>(); var foundMostRecentStackEvent = (mostRecentStackEventId == null); var request = new DescribeStackEventsRequest { StackName = stackId ?? stackName }; var eventList = new List <StackEvent>(); var ansiLinesPrinted = 0; // iterate as long as the stack is being created/updated var active = true; var success = false; while (active) { await Task.Delay(TimeSpan.FromSeconds(3)); // fetch as many events as possible for the current stack var events = new List <StackEvent>(); try { var response = await cfnClient.DescribeStackEventsAsync(request); events.AddRange(response.StackEvents); } catch (System.Net.Http.HttpRequestException e) when((e.InnerException is System.Net.Sockets.SocketException) && (e.InnerException.Message == "No such host is known")) { // ignore network issues and just try again continue; } events.Reverse(); // skip any events that preceded the most recent event before the stack update operation while (!foundMostRecentStackEvent && events.Any()) { var evt = events.First(); if (evt.EventId == mostRecentStackEventId) { foundMostRecentStackEvent = true; } seenEventIds.Add(evt.EventId); events.RemoveAt(0); } if (!foundMostRecentStackEvent) { throw new ApplicationException($"unable to find starting event for stack: {stackName}"); } // report only on new events foreach (var evt in events.Where(evt => !seenEventIds.Contains(evt.EventId))) { UpdateEvent(evt); if (!seenEventIds.Add(evt.EventId)) { // we found an event we already saw in the past, no point in looking at more events break; } if (IsFinalStackEvent(evt) && (evt.LogicalResourceId == stackName)) { // event signals stack creation/update completion; time to stop active = false; success = IsSuccessfulFinalStackEvent(evt); break; } } RenderEvents(); } if (!success) { return(Stack : null, Success : false); } // describe stack and report any output values var description = await cfnClient.DescribeStacksAsync(new DescribeStacksRequest { StackName = stackName }); return(Stack : description.Stacks.FirstOrDefault(), Success : success); // local function string TranslateLogicalIdToFullName(string logicalId) { var fullName = logicalId; nameMappings?.ResourceNameMappings?.TryGetValue(logicalId, out fullName); return(fullName ?? logicalId); } string TranslateResourceTypeToFullName(string awsType) { var fullName = awsType; nameMappings?.TypeNameMappings?.TryGetValue(awsType, out fullName); return(fullName ?? awsType); } void RenderEvents() { if (Settings.UseAnsiConsole) { if (ansiLinesPrinted > 0) { Console.Write(AnsiTerminal.MoveLineUp(ansiLinesPrinted)); } var maxResourceStatusLength = eventList.Any() ? eventList.Max(evt => evt.ResourceStatus.ToString().Length) : 0; var maxResourceTypeNameLength = eventList.Any() ? eventList.Max(evt => TranslateResourceTypeToFullName(evt.ResourceType).Length) : 0; foreach (var evt in eventList) { var resourceStatus = evt.ResourceStatus.ToString(); var resourceType = TranslateResourceTypeToFullName(evt.ResourceType); if (_ansiStatusColorCodes.TryGetValue(evt.ResourceStatus, out var ansiColor)) { // print resource status Console.Write(ansiColor); Console.Write(resourceStatus); Console.Write(AnsiTerminal.Reset); Console.Write("".PadRight(maxResourceStatusLength - resourceStatus.Length + 4)); // print resource type Console.Write(resourceType); Console.Write("".PadRight(maxResourceTypeNameLength - resourceType.Length + 4)); // print resource name Console.Write(TranslateLogicalIdToFullName(evt.LogicalResourceId)); // print status reason if ((logError == null) && (evt.ResourceStatusReason != null)) { Console.Write($" ({evt.ResourceStatusReason})"); } } else { Console.Write($"{resourceStatus} {resourceType} {TranslateLogicalIdToFullName(evt.LogicalResourceId)}{(evt.ResourceStatusReason != null ? $" ({evt.ResourceStatusReason})" : "")}"); } Console.Write(AnsiTerminal.ClearEndOfLine); Console.WriteLine(); } ansiLinesPrinted = eventList.Count; } } void UpdateEvent(StackEvent evt) { if (Settings.UseAnsiConsole) { var index = eventList.FindIndex(e => e.LogicalResourceId == evt.LogicalResourceId); if (index < 0) { eventList.Add(evt); } else { eventList[index] = evt; } } else { Console.WriteLine($"{evt.ResourceStatus,-35} {TranslateResourceTypeToFullName(evt.ResourceType),-55} {TranslateLogicalIdToFullName(evt.LogicalResourceId)}{(evt.ResourceStatusReason != null ? $" ({evt.ResourceStatusReason})" : "")}"); } // capture failed operation as an error switch (evt.ResourceStatus) { case "CREATE_FAILED": case "ROLLBACK_FAILED": case "UPDATE_FAILED": case "DELETE_FAILED": case "UPDATE_ROLLBACK_FAILED": case "UPDATE_ROLLBACK_IN_PROGRESS": if (evt.ResourceStatusReason != "Resource creation cancelled") { logError?.Invoke($"{evt.ResourceStatus} {TranslateLogicalIdToFullName(evt.LogicalResourceId)} [{TranslateResourceTypeToFullName(evt.ResourceType)}]: {evt.ResourceStatusReason}", /*Exception*/ null); } break; } } }
//--- Methods --- public async Task <bool> DeployChangeSetAsync( ModuleManifest manifest, ModuleNameMappings nameMappings, ModuleLocation moduleLocation, string stackName, bool allowDataLoss, bool protectStack, List <CloudFormationParameter> parameters ) { var now = DateTime.UtcNow; // check if cloudformation stack already exists and is in a final state Console.WriteLine(); Console.WriteLine($"Deploying stack: {stackName} [{moduleLocation.ModuleInfo}]"); var mostRecentStackEventId = await Settings.CfnClient.GetMostRecentStackEventIdAsync(stackName); // validate template (must have been copied to deployment bucket at this stage) if (moduleLocation.SourceBucketName != Settings.DeploymentBucketName) { LogError($"module source must match the deployment tier S3 bucket (EXPECTED: {Settings.DeploymentBucketName}, FOUND: {moduleLocation.SourceBucketName})"); return(false); } ValidateTemplateResponse validation; try { validation = await Settings.CfnClient.ValidateTemplateAsync(new ValidateTemplateRequest { TemplateURL = moduleLocation.ModuleTemplateUrl }); } catch (AmazonCloudFormationException e) { LogError($"{e.Message} (url: {moduleLocation.ModuleTemplateUrl})"); return(false); } // create change-set var success = false; var changeSetName = $"{moduleLocation.ModuleInfo.FullName.Replace(".", "-")}-{now:yyyy-MM-dd-hh-mm-ss}"; var updateOrCreate = (mostRecentStackEventId != null) ? "update" : "create"; var capabilities = validation.Capabilities.Any() ? "[" + string.Join(", ", validation.Capabilities) + "]" : ""; Console.WriteLine($"=> Stack {updateOrCreate} initiated for {stackName} {capabilities}"); var response = await Settings.CfnClient.CreateChangeSetAsync(new CreateChangeSetRequest { Capabilities = validation.Capabilities, ChangeSetName = changeSetName, ChangeSetType = (mostRecentStackEventId != null) ? ChangeSetType.UPDATE : ChangeSetType.CREATE, Description = $"Stack {updateOrCreate} {moduleLocation.ModuleInfo.FullName} (v{moduleLocation.ModuleInfo.Version})", Parameters = new List <CloudFormationParameter>(parameters) { new CloudFormationParameter { ParameterKey = "DeploymentPrefix", ParameterValue = Settings.TierPrefix }, new CloudFormationParameter { ParameterKey = "DeploymentPrefixLowercase", ParameterValue = Settings.TierPrefix.ToLowerInvariant() }, new CloudFormationParameter { ParameterKey = "DeploymentBucketName", ParameterValue = Settings.DeploymentBucketName } }, StackName = stackName, TemplateURL = moduleLocation.ModuleTemplateUrl, Tags = Settings.GetCloudFormationStackTags(moduleLocation.ModuleInfo.FullName, stackName) }); try { var changes = await WaitForChangeSetAsync(response.Id); if (changes == null) { return(false); } if (!changes.Any()) { Console.WriteLine("=> No stack update required"); return(true); } // changes if (!allowDataLoss) { var lossyChanges = DetectLossyChanges(changes); if (lossyChanges.Any()) { LogError("one or more resources could be replaced or deleted; use --allow-data-loss to proceed"); Console.WriteLine("=> WARNING: detected potential replacement and data-loss in the following resources"); foreach (var lossy in lossyChanges) { Console.WriteLine($"{(lossy.ResourceChange.Replacement == Replacement.True ? "ALWAYS" : "CONDITIONAL"),-12} {lossy.ResourceChange.ResourceType,-55} {TranslateLogicalIdToFullName(lossy.ResourceChange.LogicalResourceId)}"); } return(false); } } // execute change-set await Settings.CfnClient.ExecuteChangeSetAsync(new ExecuteChangeSetRequest { ChangeSetName = changeSetName, StackName = stackName }); var outcome = await Settings.CfnClient.TrackStackUpdateAsync(stackName, response.StackId, mostRecentStackEventId, nameMappings, LogError); if (outcome.Success) { Console.WriteLine($"=> Stack {updateOrCreate} finished"); ShowStackResult(outcome.Stack); success = true; } else { Console.WriteLine($"=> Stack {updateOrCreate} FAILED"); } // optionally enable stack protection if (success) { // on success, protect the stack if requested if (protectStack) { await Settings.CfnClient.UpdateTerminationProtectionAsync(new UpdateTerminationProtectionRequest { EnableTerminationProtection = protectStack, StackName = stackName }); } } else if (mostRecentStackEventId == null) { // delete a new stack that failed to create try { await Settings.CfnClient.DeleteStackAsync(new DeleteStackRequest { StackName = stackName }); } catch { } } return(success); } finally { try { await Settings.CfnClient.DeleteChangeSetAsync(new DeleteChangeSetRequest { ChangeSetName = response.Id }); } catch { } } // local function string TranslateLogicalIdToFullName(string logicalId) { var fullName = logicalId; nameMappings?.ResourceNameMappings.TryGetValue(logicalId, out fullName); return(fullName ?? logicalId); } }