/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to
/// provide specific behavior for this base class.
/// </summary>
/// <remarks>
/// This method cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
// read stream into memory
LogInfo("reading stream body");
string snsEventBody;
using (var reader = new StreamReader(stream)) {
snsEventBody = reader.ReadToEnd();
}
var stopwatch = Stopwatch.StartNew();
var metrics = new List <LambdaMetric>();
// process received sns record (there is only ever one)
try {
// sns event deserialization
LogInfo("deserializing SNS event");
try {
var snsEvent = LambdaSerializer.Deserialize <SNSEvent>(snsEventBody);
_currentRecord = snsEvent.Records.First().Sns;
// message deserialization
LogInfo("deserializing message");
var message = Deserialize(CurrentRecord.Message);
// process message
LogInfo("processing message");
await ProcessMessageAsync(message);
// record successful processing metrics
stopwatch.Stop();
var now = DateTimeOffset.UtcNow;
metrics.Add(("MessageSuccess.Count", 1, LambdaMetricUnit.Count));
metrics.Add(("MessageSuccess.Latency", stopwatch.Elapsed.TotalMilliseconds, LambdaMetricUnit.Milliseconds));
metrics.Add(("MessageSuccess.Lifespan", (now - CurrentRecord.GetLifespanTimestamp()).TotalSeconds, LambdaMetricUnit.Seconds));
return("Ok".ToStream());
} catch (Exception e) {
LogError(e);
try {
// attempt to send failed message to the dead-letter queue
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.SQS, LambdaSerializer.Serialize(snsEventBody), e);
// record failed processing metrics
metrics.Add(("MessageDead.Count", 1, LambdaMetricUnit.Count));
} catch {
// NOTE (2020-04-22, bjorg): since the message could not be sent to the dead-letter queue,
// the next best action is to let Lambda retry it; unfortunately, there is no way
// of knowing how many attempts have occurred already.
// unable to forward message to dead-letter queue; report failure to lambda so it can retry
metrics.Add(("MessageFailed.Count", 1, LambdaMetricUnit.Count));
throw;
}
return($"ERROR: {e.Message}".ToStream());
}
} finally {
_currentRecord = null;
LogMetric(metrics);
}
}
public void LambdaExpressionShouldSerialize(LambdaExpression lambda)
{
var target = lambdaSerializer.Serialize(lambda, new SerializationState());
Assert.Equal((ExpressionType)target.Type, lambda.NodeType);
}
private async Task WriteResponse(
CloudFormationResourceRequest <TProperties> rawRequest,
CloudFormationResourceResponse <TAttributes> rawResponse
)
{
Exception exception = null;
var backoff = TimeSpan.FromMilliseconds(100);
// write response to pre-signed S3 URL
for (var i = 0; i < MAX_SEND_ATTEMPTS; ++i)
{
try {
if (rawRequest.ResponseURL == null)
{
throw new InvalidOperationException("ResponseURL is missing");
}
var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage {
RequestUri = new Uri(rawRequest.ResponseURL),
Method = HttpMethod.Put,
Content = new ByteArrayContent(Encoding.UTF8.GetBytes(LambdaSerializer.Serialize(rawResponse)))
});
if (httpResponse.StatusCode != HttpStatusCode.OK)
{
throw new LambdaCustomResourceException(
"PUT operation to pre-signed S3 URL failed with status code: {0} [{1} {2}] = {3}",
httpResponse.StatusCode,
rawRequest.RequestType,
rawRequest.ResourceType ?? "<MISSING>",
await httpResponse.Content.ReadAsStringAsync()
);
}
return;
} catch (InvalidOperationException e) {
exception = e;
break;
} catch (Exception e) {
exception = e;
LogErrorAsWarning(e, "writing response to pre-signed S3 URL failed");
await Task.Delay(backoff);
backoff = TimeSpan.FromSeconds(backoff.TotalSeconds * 2);
}
}
if (exception == null)
{
exception = new ShouldNeverHappenException($"ALambdaCustomResourceFunction.WriteResponse failed w/o an explicit");
}
// max attempts have been reached; fail permanently and record the failed request for playback
LogError(exception);
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.CloudFormation, LambdaSerializer.Serialize(rawRequest), exception);
}
/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to
/// provide specific behavior for this base class.
/// </summary>
/// <remarks>
/// This method cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
// deserialize stream to sqs event
LogInfo("deserializing stream to SQS event");
var sqsEvent = LambdaSerializer.Deserialize <SQSEvent>(stream);
if (!sqsEvent.Records.Any())
{
return($"empty batch".ToStream());
}
// process all received sqs records
var eventSourceArn = sqsEvent.Records.First().EventSourceArn;
var successfulMessages = new List <SQSEvent.SQSMessage>();
foreach (var record in sqsEvent.Records)
{
_currentRecord = record;
var metrics = new List <LambdaMetric>();
try {
var stopwatch = Stopwatch.StartNew();
// attempt to deserialize the sqs record
LogInfo("deserializing message");
var message = Deserialize(record.Body);
// attempt to process the sqs message
LogInfo("processing message");
await ProcessMessageAsync(message);
successfulMessages.Add(record);
// record successful processing metrics
stopwatch.Stop();
var now = DateTimeOffset.UtcNow;
metrics.Add(("MessageSuccess.Count", 1, LambdaMetricUnit.Count));
metrics.Add(("MessageSuccess.Latency", stopwatch.Elapsed.TotalMilliseconds, LambdaMetricUnit.Milliseconds));
metrics.Add(("MessageSuccess.Lifespan", (now - record.GetLifespanTimestamp()).TotalSeconds, LambdaMetricUnit.Seconds));
} catch (Exception e) {
// NOTE (2020-04-21, bjorg): delete message if error is not retriable (i.e. logic error) or
// the message has reached it's maximum number of retries.
var deleteMessage = !(e is LambdaRetriableException) ||
(record.GetApproximateReceiveCount() >= await Provider.GetMaxRetriesForQueueAsync(record.EventSourceArn));
// the intent is to delete the message
if (deleteMessage)
{
// NOTE (2020-04-22, bjorg): always log an error since the intent is to send
// this message to the dead-letter queue.
LogError(e);
try {
// attempt to send failed message to the dead-letter queue
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.SQS, LambdaSerializer.Serialize(record), e);
// record forwarded message as successful so it gets deleted from the queue
successfulMessages.Add(record);
// record failed processing metrics
metrics.Add(("MessageDead.Count", 1, LambdaMetricUnit.Count));
} catch {
// record attempted processing metrics
metrics.Add(("MessageFailed.Count", 1, LambdaMetricUnit.Count));
}
}
else
{
// record attempted processing metrics
metrics.Add(("MessageFailed.Count", 1, LambdaMetricUnit.Count));
// log error as a warning as we expect to see this message again
LogErrorAsWarning(e);
}
} finally {
_currentRecord = null;
LogMetric(metrics);
}
}
// check if any failures occurred
if ((sqsEvent.Records.Count != successfulMessages.Count) && (successfulMessages.Count > 0))
{
// delete all messages that were successfully processed to avoid them being tried again
await Provider.DeleteMessagesFromQueueAsync(
eventSourceArn,
successfulMessages.Select(message =>
(MessageId: message.MessageId, ReceiptHandle: message.ReceiptHandle)
)
);
// fail invocation to prevent messages from being deleted
throw new LambdaAbortException($"processing failed: {sqsEvent.Records.Count - successfulMessages.Count} errors ({successfulMessages.Count} messages succeeded)");
}
return($"processed {successfulMessages.Count} messages".ToStream());
}
