private async Task WriteResponse(
CloudFormationResourceRequest <TProperties> rawRequest,
CloudFormationResourceResponse <TAttributes> rawResponse
)
{
Exception exception = null;
var backoff = TimeSpan.FromMilliseconds(100);
// write response to pre-signed S3 URL
for (var i = 0; i < MAX_SEND_ATTEMPTS; ++i)
{
try {
if (rawRequest.ResponseURL == null)
{
throw new InvalidOperationException("ResponseURL is missing");
}
var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage {
RequestUri = new Uri(rawRequest.ResponseURL),
Method = HttpMethod.Put,
Content = new ByteArrayContent(Encoding.UTF8.GetBytes(LambdaSerializer.Serialize(rawResponse)))
});
if (httpResponse.StatusCode != HttpStatusCode.OK)
{
throw new LambdaCustomResourceException(
"PUT operation to pre-signed S3 URL failed with status code: {0} [{1} {2}] = {3}",
httpResponse.StatusCode,
rawRequest.RequestType,
rawRequest.ResourceType ?? "<MISSING>",
await httpResponse.Content.ReadAsStringAsync()
);
}
return;
} catch (InvalidOperationException e) {
exception = e;
break;
} catch (Exception e) {
exception = e;
LogErrorAsWarning(e, "writing response to pre-signed S3 URL failed");
await Task.Delay(backoff);
backoff = TimeSpan.FromSeconds(backoff.TotalSeconds * 2);
}
}
if (exception == null)
{
exception = new ShouldNeverHappenException($"ALambdaCustomResourceFunction.WriteResponse failed w/o an explicit");
}
// max attempts have been reached; fail permanently and record the failed request for playback
LogError(exception);
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.CloudFormation, LambdaSerializer.Serialize(rawRequest), exception);
}
/// <summary>
/// The <see cref="HandleFailedInitializationAsync(Stream)"/> method is only invoked when an error occurs during the
/// Lambda function initialization. This method can be overridden to provide custom behavior for how to handle such
/// failures more gracefully.
/// </summary>
/// <remarks>
/// Regardless of what this method does. Once completed, the Lambda function exits by rethrowing the original exception
/// that occurred during initialization.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
protected override async Task HandleFailedInitializationAsync(Stream stream)
{
// NOTE (2018-12-12, bjorg): function initialization failed; attempt to notify
// the CloudFormation service of the failure.
try {
var rawRequest = DeserializeStream(stream);
var rawResponse = new CloudFormationResourceResponse <TAttributes> {
Status = CloudFormationResourceResponseStatus.FAILED,
Reason = "custom resource initialization failed",
StackId = rawRequest.StackId,
RequestId = rawRequest.RequestId,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = rawRequest.PhysicalResourceId ?? "no-physical-id"
};
await WriteResponse(rawRequest, rawResponse);
} catch { }
}
private async Task WriteResponse(
CloudFormationResourceRequest <TProperties> rawRequest,
CloudFormationResourceResponse <TAttributes> rawResponse
)
{
Exception exception = null;
// write response to pre-signed S3 URL
for (var i = 0; i < MAX_SEND_ATTEMPTS; ++i)
{
try {
var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage {
RequestUri = new Uri(rawRequest.ResponseURL),
Method = HttpMethod.Put,
Content = new ByteArrayContent(Encoding.UTF8.GetBytes(SerializeJson(rawResponse)))
});
if (httpResponse.StatusCode != HttpStatusCode.OK)
{
throw new LambdaCustomResourceException(
"PUT operation to pre-signed S3 URL failed with status code: {0} [{1} {2}] = {3}",
httpResponse.StatusCode,
rawRequest.RequestType,
rawRequest.ResourceType,
await httpResponse.Content.ReadAsStringAsync()
);
}
return;
} catch (Exception e) {
exception = e;
LogErrorAsWarning(e, "writing response to pre-signed S3 URL failed");
await Task.Delay(TimeSpan.FromMilliseconds(100));
}
}
// max attempts have been reached; fail permanently and record the failed request for playback
LogError(exception);
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.CloudFormation, SerializeJson(rawRequest), exception);
}
//--- Methods ---
/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> method handles the communication protocol with the AWS CloudFormation service.
/// </summary>
/// <remarks>
/// This method cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
var rawRequest = DeserializeStream(stream);
// process message
CloudFormationResourceResponse <TAttributes> rawResponse;
LogInfo("processing request");
try {
LogInfo($"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation received");
var request = new Request <TProperties> {
RequestType = rawRequest.RequestType,
ResourceType = rawRequest.ResourceType,
StackId = rawRequest.StackId,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = rawRequest.PhysicalResourceId,
ResourceProperties = rawRequest.ResourceProperties,
OldResourceProperties = rawRequest.OldResourceProperties
};
// handle slack request
Task <Response <TAttributes> > responseTask;
switch (request.RequestType)
{
case RequestType.Create:
responseTask = ProcessCreateResourceAsync(request);
break;
case RequestType.Update:
responseTask = ProcessUpdateResourceAsync(request);
break;
case RequestType.Delete:
responseTask = ProcessDeleteResourceAsync(request);
break;
default:
throw new ArgumentOutOfRangeException(nameof(request.RequestType), request.RequestType, "unexpected request value");
}
// check if the custom resource logic completes before function times out
var timeout = CurrentContext.RemainingTime - TimeSpan.FromSeconds(0.5);
if (await Task.WhenAny(Task.Delay(timeout), responseTask) != responseTask)
{
// TODO (2020-07-31, bjorg): set cancellation token when available on ProcessXYSResourceAsync()
throw new TimeoutException($"custom resource operation timed out");
}
// await completed task to trigger any contained exceptions
await responseTask;
LogInfo($"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation was successful");
rawResponse = new CloudFormationResourceResponse <TAttributes> {
Status = CloudFormationResourceResponseStatus.SUCCESS,
Reason = "",
StackId = rawRequest.StackId,
RequestId = rawRequest.RequestId,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = responseTask.Result.PhysicalResourceId ?? rawRequest.PhysicalResourceId,
NoEcho = responseTask.Result.NoEcho,
Data = responseTask.Result.Attributes
};
} catch (AggregateException aggregateException) when(
(aggregateException.InnerExceptions.Count() == 1) &&
(aggregateException.InnerExceptions[0] is CustomResourceAbortException e)
)
{
LogInfo($"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation ABORTED [{{0}}]", e.Message);
rawResponse = new CloudFormationResourceResponse <TAttributes> {
Status = CloudFormationResourceResponseStatus.FAILED,
Reason = e.Message,
StackId = rawRequest.StackId,
RequestId = rawRequest.RequestId,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = rawRequest.PhysicalResourceId ?? "no-physical-id"
};
} catch (CustomResourceAbortException e) {
LogInfo($"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation ABORTED [{{0}}]", e.Message);
rawResponse = new CloudFormationResourceResponse <TAttributes> {
Status = CloudFormationResourceResponseStatus.FAILED,
Reason = e.Message,
StackId = rawRequest.StackId,
RequestId = rawRequest.RequestId,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = rawRequest.PhysicalResourceId ?? "no-physical-id"
};
} catch (Exception e) {
LogError(e, $"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation FAILED [{{0}}]", e.Message);
rawResponse = new CloudFormationResourceResponse <TAttributes> {
Status = CloudFormationResourceResponseStatus.FAILED,
Reason = "internal error",
StackId = rawRequest.StackId,
RequestId = rawRequest.RequestId,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = rawRequest.PhysicalResourceId ?? "no-physical-id"
};
}
// write response
await WriteResponse(rawRequest, rawResponse);
return(rawResponse.Status.ToString().ToStream());
}
//--- Methods ---
public override async Task ProcessMessageAsync(CloudFormationResourceRequest <TRequestProperties> rawRequest, ILambdaContext context)
{
LogInfo(JsonConvert.SerializeObject(rawRequest, Formatting.Indented));
CloudFormationResourceResponse <TResponseProperties> rawResponse;
LogInfo($"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation received");
try {
var request = new Request <TRequestProperties> {
RequestType = rawRequest.RequestType,
ResourceType = rawRequest.ResourceType,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = rawRequest.PhysicalResourceId,
ResourceProperties = rawRequest.ResourceProperties,
OldResourceProperties = rawRequest.OldResourceProperties
};
// handle slack request
Response <TResponseProperties> response;
switch (request.RequestType)
{
case RequestType.Create:
response = await HandleCreateResourceAsync(request);
break;
case RequestType.Update:
response = await HandleUpdateResourceAsync(request);
break;
case RequestType.Delete:
response = await HandleDeleteResourceAsync(request);
break;
default:
throw new ArgumentOutOfRangeException(nameof(request.RequestType), request.RequestType, "unexpected request value");
}
rawResponse = new CloudFormationResourceResponse <TResponseProperties> {
Status = CloudFormationResourceResponseStatus.SUCCESS,
Reason = "",
StackId = rawRequest.StackId,
RequestId = rawRequest.RequestId,
LogicalResourceId = rawRequest.LogicalResourceId,
PhysicalResourceId = response.PhysicalResourceId ?? rawRequest.PhysicalResourceId,
NoEcho = response.NoEcho,
Data = response.Properties
};
LogInfo($"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation was successful");
} catch (Exception e) {
LogError(e, $"{rawRequest.ResourceType}: {rawRequest.RequestType.ToString().ToUpperInvariant()} operation FAILED [{{0}}]", e.Message);
rawResponse = new CloudFormationResourceResponse <TResponseProperties> {
Status = CloudFormationResourceResponseStatus.FAILED,
Reason = e.Message,
StackId = rawRequest.StackId,
RequestId = rawRequest.RequestId,
LogicalResourceId = rawRequest.LogicalResourceId
};
}
// write response to pre-signed S3 URL
try {
var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage {
RequestUri = new Uri(rawRequest.ResponseURL),
Method = HttpMethod.Put,
Content = new ByteArrayContent(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(rawResponse)))
});
if (httpResponse.StatusCode != HttpStatusCode.OK)
{
throw new CustomResourceException(
"PUT operation to pre-signed S3 URL failed with status code: {0} [{1} {2}] = {3}",
httpResponse.StatusCode,
rawRequest.RequestType,
rawRequest.ResourceType,
await httpResponse.Content.ReadAsStringAsync()
);
}
} catch (Exception e) {
LogError(e, "writing response to pre-signed S3 URL failed");
// TODO (2018-06-14, bjorg): how should we handle this? we may want the handler to re-attempt the
// resource creation since the request was SNS based
throw;
}
}
