//--- Methods ---
public async Task <bool> DeployChangeSetAsync(
ModuleManifest manifest,
ModuleNameMappings nameMappings,
ModuleLocation moduleLocation,
string stackName,
bool allowDataLoss,
bool protectStack,
List <CloudFormationParameter> parameters
)
{
var now = DateTime.UtcNow;
// check if cloudformation stack already exists and is in a final state
Console.WriteLine();
Console.WriteLine($"Deploying stack: {stackName} [{moduleLocation.ModuleInfo}]");
var mostRecentStackEventId = await Settings.CfnClient.GetMostRecentStackEventIdAsync(stackName);
// validate template (must have been copied to deployment bucket at this stage)
if (moduleLocation.SourceBucketName != Settings.DeploymentBucketName)
{
LogError($"module source must match the deployment tier S3 bucket (EXPECTED: {Settings.DeploymentBucketName}, FOUND: {moduleLocation.SourceBucketName})");
return(false);
}
ValidateTemplateResponse validation;
try {
validation = await Settings.CfnClient.ValidateTemplateAsync(new ValidateTemplateRequest {
TemplateURL = moduleLocation.ModuleTemplateUrl
});
} catch (AmazonCloudFormationException e) {
LogError($"{e.Message} (url: {moduleLocation.ModuleTemplateUrl})");
return(false);
}
// verify if we need to remove the old CloudFormation notification ARNs that were created by LambdaSharp config before v0.7
List <string> notificationARNs = null;
ModuleNameMappings oldNameMappings = null;
if (mostRecentStackEventId != null)
{
// fetch name mappings for current template; this is needed to properly map logical IDs to their original names when they get deleted
oldNameMappings = await new ModelManifestLoader(Settings, "source").GetNameMappingsFromCloudFormationStackAsync(stackName);
// NOTE (2019-09-19, bjorg): this is a HACK to remove the old notification ARNs, because doing it part
// of the change set is not working for some reason.
var deployedModule = await Settings.CfnClient.GetStackAsync(stackName, LogError);
notificationARNs = deployedModule.Stack.NotificationARNs.Where(arn => !arn.Contains(":LambdaSharpTool-")).ToList();
if (notificationARNs.Count != deployedModule.Stack.NotificationARNs.Count)
{
Console.WriteLine("=> Removing legacy stack notification ARN");
var update = await Settings.CfnClient.UpdateStackAsync(new UpdateStackRequest {
StackName = stackName,
UsePreviousTemplate = true,
Parameters = deployedModule.Stack.Parameters.Select(param => new CloudFormationParameter {
ParameterKey = param.ParameterKey,
UsePreviousValue = true
}).ToList(),
NotificationARNs = notificationARNs,
Capabilities = validation.Capabilities
});
var outcome = await Settings.CfnClient.TrackStackUpdateAsync(stackName, update.StackId, mostRecentStackEventId, nameMappings, oldNameMappings, LogError);
if (!outcome.Success)
{
LogError("failed to remove legacy stack notification ARN; remove manually and try again");
return(false);
}
mostRecentStackEventId = await Settings.CfnClient.GetMostRecentStackEventIdAsync(stackName);
Console.WriteLine("=> Legacy stack notification ARN has been removed");
}
}
// create change-set
var success = false;
var changeSetName = $"{moduleLocation.ModuleInfo.FullName.Replace(".", "-")}-{now:yyyy-MM-dd-hh-mm-ss}";
var updateOrCreate = (mostRecentStackEventId != null) ? "update" : "create";
var capabilities = validation.Capabilities.Any()
? "[" + string.Join(", ", validation.Capabilities) + "]"
: "";
Console.WriteLine($"=> Stack {updateOrCreate} initiated for {Settings.InfoColor}{stackName}{Settings.ResetColor} {capabilities}");
CreateChangeSetResponse response;
try {
response = await Settings.CfnClient.CreateChangeSetAsync(new CreateChangeSetRequest {
Capabilities = validation.Capabilities,
ChangeSetName = changeSetName,
ChangeSetType = (mostRecentStackEventId != null) ? ChangeSetType.UPDATE : ChangeSetType.CREATE,
Description = $"Stack {updateOrCreate} {moduleLocation.ModuleInfo.FullName} (v{moduleLocation.ModuleInfo.Version})",
Parameters = new List <CloudFormationParameter>(parameters)
{
new CloudFormationParameter {
ParameterKey = "DeploymentPrefix",
ParameterValue = Settings.TierPrefix
},
new CloudFormationParameter {
ParameterKey = "DeploymentPrefixLowercase",
ParameterValue = Settings.TierPrefix.ToLowerInvariant()
},
new CloudFormationParameter {
ParameterKey = "DeploymentBucketName",
ParameterValue = Settings.DeploymentBucketName
},
new CloudFormationParameter {
ParameterKey = "DeploymentChecksum",
ParameterValue = manifest.TemplateChecksum
}
},
StackName = stackName,
TemplateURL = moduleLocation.ModuleTemplateUrl,
Tags = Settings.GetCloudFormationStackTags(moduleLocation.ModuleInfo.FullName, stackName)
});
} catch (AmazonCloudFormationException e) {
LogError($"cloudformation change-set failed: {e.Message}");
return(false);
}
try {
var changes = await WaitForChangeSetAsync(response.Id);
if (changes == null)
{
return(false);
}
if (!changes.Any())
{
Console.WriteLine("=> No stack update required");
return(true);
}
// changes
if (!allowDataLoss)
{
var lossyChanges = DetectLossyChanges(changes);
if (lossyChanges.Any())
{
Console.WriteLine();
Console.WriteLine($"{Settings.AlertColor}CAUTION:{Settings.ResetColor} detected potential replacement and data-loss in the following resources");
var maxResourceTypeWidth = lossyChanges.Select(change => change.ResourceChange.ResourceType.Length).Max();
foreach (var lossy in lossyChanges)
{
if (Settings.UseAnsiConsole)
{
if (lossy.ResourceChange.Replacement == Replacement.True)
{
Console.Write(AnsiTerminal.Red);
Console.Write("ALWAYS ");
}
else
{
Console.Write(AnsiTerminal.Yellow);
Console.Write("CONDITIONAL ");
}
Console.Write(AnsiTerminal.Reset);
}
else
{
Console.WriteLine((lossy.ResourceChange.Replacement == Replacement.True)
? "ALWAYS "
: "CONDITIONAL "
);
}
Console.Write(lossy.ResourceChange.ResourceType);
Console.Write("".PadRight(maxResourceTypeWidth - lossy.ResourceChange.ResourceType.Length + 4));
Console.Write(TranslateLogicalIdToFullName(lossy.ResourceChange.LogicalResourceId));
Console.WriteLine();
}
if (!Settings.PromptYesNo("Proceed with potentially replacing/deleting resources?", false))
{
return(false);
}
Console.WriteLine();
}
}
// execute change-set
await Settings.CfnClient.ExecuteChangeSetAsync(new ExecuteChangeSetRequest {
ChangeSetName = changeSetName,
StackName = stackName
});
var outcome = await Settings.CfnClient.TrackStackUpdateAsync(
stackName,
response.StackId,
mostRecentStackEventId,
nameMappings,
oldNameMappings,
LogError
);
if (outcome.Success)
{
Console.WriteLine($"=> Stack {updateOrCreate} finished");
ShowStackResult(outcome.Stack);
success = true;
}
else
{
Console.WriteLine($"=> Stack {updateOrCreate} FAILED");
}
// optionally enable stack protection
if (success)
{
// on success, protect the stack if requested
if (protectStack)
{
await Settings.CfnClient.UpdateTerminationProtectionAsync(new UpdateTerminationProtectionRequest {
EnableTerminationProtection = protectStack,
StackName = stackName
});
}
}
else if (mostRecentStackEventId == null)
{
// delete a new stack that failed to create
try {
await Settings.CfnClient.DeleteStackAsync(new DeleteStackRequest {
StackName = stackName
});
} catch { }
}
return(success);
} finally {
try {
await Settings.CfnClient.DeleteChangeSetAsync(new DeleteChangeSetRequest {
ChangeSetName = response.Id
});
} catch { }
}
// local function
string TranslateLogicalIdToFullName(string logicalId)
{
var fullName = logicalId;
nameMappings?.ResourceNameMappings.TryGetValue(logicalId, out fullName);
return(fullName ?? logicalId);
}
}
public static async Task <(Stack Stack, bool Success)> TrackStackUpdateAsync(
this IAmazonCloudFormation cfnClient,
string stackName,
string stackId,
string mostRecentStackEventId,
ModuleNameMappings nameMappings = null,
LogErrorDelegate logError = null
)
{
var seenEventIds = new HashSet <string>();
var foundMostRecentStackEvent = (mostRecentStackEventId == null);
var request = new DescribeStackEventsRequest {
StackName = stackId ?? stackName
};
var eventList = new List <StackEvent>();
var ansiLinesPrinted = 0;
// iterate as long as the stack is being created/updated
var active = true;
var success = false;
while (active)
{
await Task.Delay(TimeSpan.FromSeconds(3));
// fetch as many events as possible for the current stack
var events = new List <StackEvent>();
try {
var response = await cfnClient.DescribeStackEventsAsync(request);
events.AddRange(response.StackEvents);
} catch (System.Net.Http.HttpRequestException e) when((e.InnerException is System.Net.Sockets.SocketException) && (e.InnerException.Message == "No such host is known"))
{
// ignore network issues and just try again
continue;
}
events.Reverse();
// skip any events that preceded the most recent event before the stack update operation
while (!foundMostRecentStackEvent && events.Any())
{
var evt = events.First();
if (evt.EventId == mostRecentStackEventId)
{
foundMostRecentStackEvent = true;
}
seenEventIds.Add(evt.EventId);
events.RemoveAt(0);
}
if (!foundMostRecentStackEvent)
{
throw new ApplicationException($"unable to find starting event for stack: {stackName}");
}
// report only on new events
foreach (var evt in events.Where(evt => !seenEventIds.Contains(evt.EventId)))
{
UpdateEvent(evt);
if (!seenEventIds.Add(evt.EventId))
{
// we found an event we already saw in the past, no point in looking at more events
break;
}
if (IsFinalStackEvent(evt) && (evt.LogicalResourceId == stackName))
{
// event signals stack creation/update completion; time to stop
active = false;
success = IsSuccessfulFinalStackEvent(evt);
break;
}
}
RenderEvents();
}
if (!success)
{
return(Stack : null, Success : false);
}
// describe stack and report any output values
var description = await cfnClient.DescribeStacksAsync(new DescribeStacksRequest {
StackName = stackName
});
return(Stack : description.Stacks.FirstOrDefault(), Success : success);
// local function
string TranslateLogicalIdToFullName(string logicalId)
{
var fullName = logicalId;
nameMappings?.ResourceNameMappings?.TryGetValue(logicalId, out fullName);
return(fullName ?? logicalId);
}
string TranslateResourceTypeToFullName(string awsType)
{
var fullName = awsType;
nameMappings?.TypeNameMappings?.TryGetValue(awsType, out fullName);
return(fullName ?? awsType);
}
void RenderEvents()
{
if (Settings.UseAnsiConsole)
{
if (ansiLinesPrinted > 0)
{
Console.Write(AnsiTerminal.MoveLineUp(ansiLinesPrinted));
}
var maxResourceStatusLength = eventList.Any() ? eventList.Max(evt => evt.ResourceStatus.ToString().Length) : 0;
var maxResourceTypeNameLength = eventList.Any() ? eventList.Max(evt => TranslateResourceTypeToFullName(evt.ResourceType).Length) : 0;
foreach (var evt in eventList)
{
var resourceStatus = evt.ResourceStatus.ToString();
var resourceType = TranslateResourceTypeToFullName(evt.ResourceType);
if (_ansiStatusColorCodes.TryGetValue(evt.ResourceStatus, out var ansiColor))
{
// print resource status
Console.Write(ansiColor);
Console.Write(resourceStatus);
Console.Write(AnsiTerminal.Reset);
Console.Write("".PadRight(maxResourceStatusLength - resourceStatus.Length + 4));
// print resource type
Console.Write(resourceType);
Console.Write("".PadRight(maxResourceTypeNameLength - resourceType.Length + 4));
// print resource name
Console.Write(TranslateLogicalIdToFullName(evt.LogicalResourceId));
// print status reason
if ((logError == null) && (evt.ResourceStatusReason != null))
{
Console.Write($" ({evt.ResourceStatusReason})");
}
}
else
{
Console.Write($"{resourceStatus} {resourceType} {TranslateLogicalIdToFullName(evt.LogicalResourceId)}{(evt.ResourceStatusReason != null ? $" ({evt.ResourceStatusReason})" : "")}");
}
Console.Write(AnsiTerminal.ClearEndOfLine);
Console.WriteLine();
}
ansiLinesPrinted = eventList.Count;
}
}
void UpdateEvent(StackEvent evt)
{
if (Settings.UseAnsiConsole)
{
var index = eventList.FindIndex(e => e.LogicalResourceId == evt.LogicalResourceId);
if (index < 0)
{
eventList.Add(evt);
}
else
{
eventList[index] = evt;
}
}
else
{
Console.WriteLine($"{evt.ResourceStatus,-35} {TranslateResourceTypeToFullName(evt.ResourceType),-55} {TranslateLogicalIdToFullName(evt.LogicalResourceId)}{(evt.ResourceStatusReason != null ? $" ({evt.ResourceStatusReason})" : "")}");
}
// capture failed operation as an error
switch (evt.ResourceStatus)
{
case "CREATE_FAILED":
case "ROLLBACK_FAILED":
case "UPDATE_FAILED":
case "DELETE_FAILED":
case "UPDATE_ROLLBACK_FAILED":
case "UPDATE_ROLLBACK_IN_PROGRESS":
if (evt.ResourceStatusReason != "Resource creation cancelled")
{
logError?.Invoke($"{evt.ResourceStatus} {TranslateLogicalIdToFullName(evt.LogicalResourceId)} [{TranslateResourceTypeToFullName(evt.ResourceType)}]: {evt.ResourceStatusReason}", /*Exception*/ null);
}
break;
}
}
}
//--- Methods ---
public async Task <bool> DeployChangeSetAsync(
ModuleManifest manifest,
ModuleNameMappings nameMappings,
ModuleLocation moduleLocation,
string stackName,
bool allowDataLoss,
bool protectStack,
List <CloudFormationParameter> parameters
)
{
var now = DateTime.UtcNow;
// check if cloudformation stack already exists and is in a final state
Console.WriteLine();
Console.WriteLine($"Deploying stack: {stackName} [{moduleLocation.ModuleInfo}]");
var mostRecentStackEventId = await Settings.CfnClient.GetMostRecentStackEventIdAsync(stackName);
// validate template (must have been copied to deployment bucket at this stage)
if (moduleLocation.SourceBucketName != Settings.DeploymentBucketName)
{
LogError($"module source must match the deployment tier S3 bucket (EXPECTED: {Settings.DeploymentBucketName}, FOUND: {moduleLocation.SourceBucketName})");
return(false);
}
ValidateTemplateResponse validation;
try {
validation = await Settings.CfnClient.ValidateTemplateAsync(new ValidateTemplateRequest {
TemplateURL = moduleLocation.ModuleTemplateUrl
});
} catch (AmazonCloudFormationException e) {
LogError($"{e.Message} (url: {moduleLocation.ModuleTemplateUrl})");
return(false);
}
// create change-set
var success = false;
var changeSetName = $"{moduleLocation.ModuleInfo.FullName.Replace(".", "-")}-{now:yyyy-MM-dd-hh-mm-ss}";
var updateOrCreate = (mostRecentStackEventId != null) ? "update" : "create";
var capabilities = validation.Capabilities.Any()
? "[" + string.Join(", ", validation.Capabilities) + "]"
: "";
Console.WriteLine($"=> Stack {updateOrCreate} initiated for {stackName} {capabilities}");
var response = await Settings.CfnClient.CreateChangeSetAsync(new CreateChangeSetRequest {
Capabilities = validation.Capabilities,
ChangeSetName = changeSetName,
ChangeSetType = (mostRecentStackEventId != null) ? ChangeSetType.UPDATE : ChangeSetType.CREATE,
Description = $"Stack {updateOrCreate} {moduleLocation.ModuleInfo.FullName} (v{moduleLocation.ModuleInfo.Version})",
Parameters = new List <CloudFormationParameter>(parameters)
{
new CloudFormationParameter {
ParameterKey = "DeploymentPrefix",
ParameterValue = Settings.TierPrefix
},
new CloudFormationParameter {
ParameterKey = "DeploymentPrefixLowercase",
ParameterValue = Settings.TierPrefix.ToLowerInvariant()
},
new CloudFormationParameter {
ParameterKey = "DeploymentBucketName",
ParameterValue = Settings.DeploymentBucketName
}
},
StackName = stackName,
TemplateURL = moduleLocation.ModuleTemplateUrl,
Tags = Settings.GetCloudFormationStackTags(moduleLocation.ModuleInfo.FullName, stackName)
});
try {
var changes = await WaitForChangeSetAsync(response.Id);
if (changes == null)
{
return(false);
}
if (!changes.Any())
{
Console.WriteLine("=> No stack update required");
return(true);
}
// changes
if (!allowDataLoss)
{
var lossyChanges = DetectLossyChanges(changes);
if (lossyChanges.Any())
{
LogError("one or more resources could be replaced or deleted; use --allow-data-loss to proceed");
Console.WriteLine("=> WARNING: detected potential replacement and data-loss in the following resources");
foreach (var lossy in lossyChanges)
{
Console.WriteLine($"{(lossy.ResourceChange.Replacement == Replacement.True ? "ALWAYS" : "CONDITIONAL"),-12} {lossy.ResourceChange.ResourceType,-55} {TranslateLogicalIdToFullName(lossy.ResourceChange.LogicalResourceId)}");
}
return(false);
}
}
// execute change-set
await Settings.CfnClient.ExecuteChangeSetAsync(new ExecuteChangeSetRequest {
ChangeSetName = changeSetName,
StackName = stackName
});
var outcome = await Settings.CfnClient.TrackStackUpdateAsync(stackName, response.StackId, mostRecentStackEventId, nameMappings, LogError);
if (outcome.Success)
{
Console.WriteLine($"=> Stack {updateOrCreate} finished");
ShowStackResult(outcome.Stack);
success = true;
}
else
{
Console.WriteLine($"=> Stack {updateOrCreate} FAILED");
}
// optionally enable stack protection
if (success)
{
// on success, protect the stack if requested
if (protectStack)
{
await Settings.CfnClient.UpdateTerminationProtectionAsync(new UpdateTerminationProtectionRequest {
EnableTerminationProtection = protectStack,
StackName = stackName
});
}
}
else if (mostRecentStackEventId == null)
{
// delete a new stack that failed to create
try {
await Settings.CfnClient.DeleteStackAsync(new DeleteStackRequest {
StackName = stackName
});
} catch { }
}
return(success);
} finally {
try {
await Settings.CfnClient.DeleteChangeSetAsync(new DeleteChangeSetRequest {
ChangeSetName = response.Id
});
} catch { }
}
// local function
string TranslateLogicalIdToFullName(string logicalId)
{
var fullName = logicalId;
nameMappings?.ResourceNameMappings.TryGetValue(logicalId, out fullName);
return(fullName ?? logicalId);
}
}
