public void ChangeSkewBasedOnRequestEndpointAndNotConfigEndpoint()
{
Uri configEndpoint = new Uri("https://s3.amazonaws.com");
Uri requestEndpoint = new Uri("https://bucketname.s3.amazonaws.com");
ReflectionHelpers.Invoke(typeof(CorrectClockSkew), "SetClockCorrectionForEndpoint",
new object[] { configEndpoint.ToString(), TimeSpan.FromHours(-1) });
ReflectionHelpers.Invoke(typeof(CorrectClockSkew), "SetClockCorrectionForEndpoint",
new object[] { requestEndpoint.ToString(), TimeSpan.Zero });
Assert.AreEqual(TimeSpan.Zero, CorrectClockSkew.GetClockCorrectionForEndpoint(requestEndpoint.ToString()));
Tester.Reset();
Tester.Action = (int callCount) =>
{
var timeString = DateTime.UtcNow.AddHours(-1).ToString(AWSSDKUtils.ISO8601BasicDateTimeFormat);
var exception = new AmazonS3Exception("(" + timeString + " - ");
exception.ErrorCode = "RequestTimeTooSkewed";
throw exception;
};
Utils.AssertExceptionExpected(() =>
{
var request = CreateTestContext();
request.RequestContext.Request.Endpoint = requestEndpoint;
RuntimePipeline.InvokeSync(request);
},
typeof(AmazonServiceException));
// RetryPolicy should see that the clock skew for bucketname.s3.amazonaws.com is zero and change it to ~ -1 hour
Assert.IsTrue(CorrectClockSkew.GetClockCorrectionForEndpoint(requestEndpoint.ToString()) < TimeSpan.FromMinutes(-55));
}
internal async Task ConsumeDelayedMessages(ReceiveMessageRequest request, CancellationToken token)
{
var receivedMessages = await sqsClient.ReceiveMessageAsync(request, token).ConfigureAwait(false);
if (receivedMessages.Messages.Count == 0)
{
return;
}
var clockCorrection = CorrectClockSkew.GetClockCorrectionForEndpoint(awsEndpointUrl);
var preparedMessages = PrepareMessages(token, receivedMessages, clockCorrection);
token.ThrowIfCancellationRequested();
await BatchDispatchPreparedMessages(preparedMessages).ConfigureAwait(false);
}
async Task ProcessMessage(Message receivedMessage, CancellationToken token)
{
try
{
byte[] messageBody = null;
TransportMessage transportMessage = null;
Exception exception = null;
var nativeMessageId = receivedMessage.MessageId;
string messageId = null;
var isPoisonMessage = false;
try
{
if (receivedMessage.MessageAttributes.TryGetValue(Headers.MessageId, out var messageIdAttribute))
{
messageId = messageIdAttribute.StringValue;
}
else
{
messageId = nativeMessageId;
}
transportMessage = SimpleJson.DeserializeObject <TransportMessage>(receivedMessage.Body);
messageBody = await transportMessage.RetrieveBody(s3Client, configuration, token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// shutting down
return;
}
catch (Exception ex)
{
// Can't deserialize. This is a poison message
exception = ex;
isPoisonMessage = true;
}
if (isPoisonMessage || messageBody == null || transportMessage == null)
{
var logMessage = $"Treating message with {messageId} as a poison message. Moving to error queue.";
if (exception != null)
{
Logger.Warn(logMessage, exception);
}
else
{
Logger.Warn(logMessage);
}
await MovePoisonMessageToErrorQueue(receivedMessage, messageId).ConfigureAwait(false);
return;
}
if (!IsMessageExpired(receivedMessage, transportMessage.Headers, messageId, CorrectClockSkew.GetClockCorrectionForEndpoint(awsEndpointUrl)))
{
// here we also want to use the native message id because the core demands it like that
await ProcessMessageWithInMemoryRetries(transportMessage.Headers, nativeMessageId, messageBody, token).ConfigureAwait(false);
}
// Always delete the message from the queue.
// If processing failed, the onError handler will have moved the message
// to a retry queue.
await DeleteMessageAndBodyIfRequired(receivedMessage, transportMessage.S3BodyKey).ConfigureAwait(false);
}
finally
{
maxConcurrencySemaphore.Release();
}
}
async Task ConsumeDelayedMessages(ReceiveMessageRequest request, CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
var receivedMessages = await sqsClient.ReceiveMessageAsync(request, token).ConfigureAwait(false);
var clockCorrection = CorrectClockSkew.GetClockCorrectionForEndpoint(awsEndpointUrl);
foreach (var receivedMessage in receivedMessages.Messages)
{
long delaySeconds = 0;
if (receivedMessage.MessageAttributes.TryGetValue(TransportHeaders.DelaySeconds, out var delayAttribute))
{
long.TryParse(delayAttribute.StringValue, out delaySeconds);
}
var sent = receivedMessage.GetAdjustedDateTimeFromServerSetAttributes("SentTimestamp", clockCorrection);
var received = receivedMessage.GetAdjustedDateTimeFromServerSetAttributes("ApproximateFirstReceiveTimestamp", clockCorrection);
if (Convert.ToInt32(receivedMessage.Attributes["ApproximateReceiveCount"]) > 1)
{
received = DateTime.UtcNow;
}
var elapsed = received - sent;
var remainingDelay = delaySeconds - (long)elapsed.TotalSeconds;
SendMessageRequest sendMessageRequest;
if (remainingDelay > configuration.DelayedDeliveryQueueDelayTime)
{
sendMessageRequest = new SendMessageRequest(delayedDeliveryQueueUrl, receivedMessage.Body)
{
MessageAttributes =
{
[TransportHeaders.DelaySeconds] = new MessageAttributeValue
{
StringValue = remainingDelay.ToString(),
DataType = "String"
}
}
};
var deduplicationId = receivedMessage.Attributes["MessageDeduplicationId"];
// this is only here for acceptance testing purpose. In real prod code this is always false.
if (configuration.DelayedDeliveryQueueDelayTime < TransportConfiguration.AwsMaximumQueueDelayTime)
{
deduplicationId = Guid.NewGuid().ToString();
}
sendMessageRequest.MessageDeduplicationId = sendMessageRequest.MessageGroupId = deduplicationId;
}
else
{
sendMessageRequest = new SendMessageRequest(queueUrl, receivedMessage.Body);
if (remainingDelay > 0)
{
sendMessageRequest.DelaySeconds = Convert.ToInt32(remainingDelay);
}
}
try
{
await sqsClient.SendMessageAsync(sendMessageRequest, CancellationToken.None)
.ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.Debug("ConsumeDelayedMessages -> SendMessageAsync failed", ex);
await sqsClient.ChangeMessageVisibilityAsync(request.QueueUrl, receivedMessage.ReceiptHandle, 0, CancellationToken.None)
.ConfigureAwait(false);
continue;
}
try
{
await sqsClient.DeleteMessageAsync(delayedDeliveryQueueUrl, receivedMessage.ReceiptHandle, CancellationToken.None)
.ConfigureAwait(false);
}
catch (ReceiptHandleIsInvalidException ex)
{
Logger.Info($"Message receipt handle {receivedMessage.ReceiptHandle} no longer valid.", ex);
}
}
}
catch (OperationCanceledException)
{
// ignore for graceful shutdown
}
catch (Exception ex)
{
Logger.Error("Exception thrown when consuming delayed messages", ex);
}
}
}
async Task ProcessMessage(SQSEvent.SQSMessage receivedMessage, ILambdaContext lambdaContext, CancellationToken token)
{
byte[] messageBody = null;
TransportMessage transportMessage = null;
Exception exception = null;
var nativeMessageId = receivedMessage.MessageId;
string messageId = null;
var isPoisonMessage = false;
try
{
if (receivedMessage.MessageAttributes.TryGetValue(Headers.MessageId, out var messageIdAttribute))
{
messageId = messageIdAttribute.StringValue;
}
else
{
messageId = nativeMessageId;
}
transportMessage = SimpleJson.DeserializeObject <TransportMessage>(receivedMessage.Body);
messageBody = await transportMessage.RetrieveBody(s3Client, s3BucketForLargeMessages, token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
// Can't deserialize. This is a poison message
exception = ex;
isPoisonMessage = true;
}
if (isPoisonMessage || messageBody == null || transportMessage == null)
{
LogPoisonMessage(messageId, exception);
await MovePoisonMessageToErrorQueue(receivedMessage, messageId).ConfigureAwait(false);
return;
}
if (!IsMessageExpired(receivedMessage, transportMessage.Headers, messageId, CorrectClockSkew.GetClockCorrectionForEndpoint(awsEndpointUrl)))
{
// here we also want to use the native message id because the core demands it like that
await ProcessMessageWithInMemoryRetries(transportMessage.Headers, nativeMessageId, messageBody, lambdaContext, token).ConfigureAwait(false);
}
// Always delete the message from the queue.
// If processing failed, the onError handler will have moved the message
// to a retry queue.
await DeleteMessageAndBodyIfRequired(receivedMessage, transportMessage.S3BodyKey).ConfigureAwait(false);
}
async Task ConsumeDelayedMessages(ReceiveMessageRequest request, CancellationToken token)
{
while (!token.IsCancellationRequested)
{
try
{
var receivedMessages = await sqsClient.ReceiveMessageAsync(request, token).ConfigureAwait(false);
var clockCorrection = CorrectClockSkew.GetClockCorrectionForEndpoint(awsEndpointUrl);
foreach (var receivedMessage in receivedMessages.Messages)
{
long delaySeconds = 0;
if (receivedMessage.MessageAttributes.TryGetValue(TransportHeaders.DelaySeconds, out var delayAttribute))
{
long.TryParse(delayAttribute.StringValue, out delaySeconds);
}
string messageId = null;
if (receivedMessage.MessageAttributes.TryGetValue(Headers.MessageId, out var messageIdAttribute))
{
messageId = messageIdAttribute.StringValue;
}
var sent = receivedMessage.GetAdjustedDateTimeFromServerSetAttributes("SentTimestamp", clockCorrection);
var received = receivedMessage.GetAdjustedDateTimeFromServerSetAttributes("ApproximateFirstReceiveTimestamp", clockCorrection);
if (Convert.ToInt32(receivedMessage.Attributes["ApproximateReceiveCount"]) > 1)
{
received = DateTime.UtcNow;
}
var elapsed = received - sent;
var remainingDelay = delaySeconds - (long)elapsed.TotalSeconds;
SendMessageRequest sendMessageRequest;
if (remainingDelay > configuration.DelayedDeliveryQueueDelayTime)
{
sendMessageRequest = new SendMessageRequest(delayedDeliveryQueueUrl, receivedMessage.Body)
{
MessageAttributes =
{
[TransportHeaders.DelaySeconds] = new MessageAttributeValue
{
StringValue = remainingDelay.ToString(),
DataType = "String"
}
}
};
var deduplicationId = receivedMessage.Attributes["MessageDeduplicationId"];
// this is only here for acceptance testing purpose. In real prod code this is always false.
// it allows us to fake multiple cycles over the FIFO queue without being subjected to deduplication
if (configuration.DelayedDeliveryQueueDelayTime < TransportConfiguration.AwsMaximumQueueDelayTime)
{
deduplicationId = Guid.NewGuid().ToString();
}
sendMessageRequest.MessageDeduplicationId = sendMessageRequest.MessageGroupId = deduplicationId;
}
else
{
sendMessageRequest = new SendMessageRequest(queueUrl, receivedMessage.Body);
if (remainingDelay > 0)
{
sendMessageRequest.DelaySeconds = Convert.ToInt32(remainingDelay);
}
}
if (string.IsNullOrEmpty(messageId))
{
// for backward compatibility if we couldn't fetch the message id header from the attributes we use the message deduplication id
messageId = receivedMessage.Attributes["MessageDeduplicationId"];
}
// because message attributes are part of the content size restriction we want to prevent message size from changing thus we add it
// for native delayed deliver as well
sendMessageRequest.MessageAttributes[Headers.MessageId] = new MessageAttributeValue
{
StringValue = messageId,
DataType = "String"
};
try
{
await sqsClient.SendMessageAsync(sendMessageRequest, CancellationToken.None)
.ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.Debug("ConsumeDelayedMessages -> SendMessageAsync failed", ex);
await sqsClient.ChangeMessageVisibilityAsync(request.QueueUrl, receivedMessage.ReceiptHandle, 0, CancellationToken.None)
.ConfigureAwait(false);
continue;
}
try
{
await sqsClient.DeleteMessageAsync(delayedDeliveryQueueUrl, receivedMessage.ReceiptHandle, CancellationToken.None)
.ConfigureAwait(false);
}
catch (ReceiptHandleIsInvalidException ex)
{
Logger.Info($"Message receipt handle {receivedMessage.ReceiptHandle} no longer valid.", ex);
}
}
}
catch (OperationCanceledException)
{
// ignore for graceful shutdown
}
catch (Exception ex)
{
Logger.Error("Exception thrown when consuming delayed messages", ex);
}
}
}