public Task <KafkaMessage <TKey, TValue>?> ConsumeAsync(CancellationToken cancellationToken)
{
return(Task.Run(() =>
{
try
{
if (_consumer is null)
{
return null;
}
var consumeResult = _consumer.Consume(cancellationToken);
if (consumeResult is null || consumeResult.IsPartitionEOF)
{
Logger.LogInformation(consumeResult is null ? "No message received from consumer." : $"End of partition reached. Topic: {consumeResult.Topic} Partition: {consumeResult.Partition} Offset: {consumeResult.Offset}");
return null;
}
var messageProperties = ParseMessageHeaders(consumeResult, out var contentType, out var messageType);
if (string.IsNullOrWhiteSpace(contentType))
{
contentType = ContentTypes.Json;
}
var keyType = TypeCache <TKey> .AssemblyQualifiedName;
if (string.IsNullOrWhiteSpace(keyType))
{
Throw.Exception("Cannot find key assembly type name for: " + typeof(TKey).Name);
}
if (string.IsNullOrWhiteSpace(messageType))
{
Throw.Exception("Cannot find message assembly type name for: " + typeof(TKey).Name);
}
var key = _deserializationProvider.From <TKey>(consumeResult.Message.Key, contentType, keyType);
var value = _deserializationProvider.From <TValue>(consumeResult.Message.Value, contentType, messageType);
_offsetTracker.AddOffset(consumeResult.Partition, consumeResult.Offset);
return new KafkaMessage <TKey, TValue>(_acknowledgementAction, consumeResult.Partition.Value, consumeResult.Offset.Value)
{
Id = key,
Properties = messageProperties,
Value = value
};
}
catch (OperationCanceledException)
{
return null;
}
}));
}
public async Task <List <SqsMessage <T> > > ConsumeAsync(CancellationToken cancellationToken)
{
if (_currentConsumerOptions is null || _client is null)
{
Throw.Exception(MisconfiguredConsumerMessage);
}
var request = new ReceiveMessageRequest
{
QueueUrl = _currentConsumerOptions.QueueUrl,
MaxNumberOfMessages = _currentConsumerOptions.MaxNumberOfMessages,
WaitTimeSeconds = _currentConsumerOptions.WaitTimeSeconds,
AttributeNames = _currentConsumerOptions.SQSMessageAttributes,
MessageAttributeNames = _currentConsumerOptions.CustomMessageAttributes
};
if (_currentConsumerOptions.VisibilityTimeout.HasValue)
{
request.VisibilityTimeout = _currentConsumerOptions.VisibilityTimeout.Value;
}
var response = await _client.ReceiveMessageAsync(request, cancellationToken);
if (response is null || response.HttpStatusCode != HttpStatusCode.OK || response.Messages is null || response.Messages.Count == 0)
{
return(_emptyList);
}
var result = new List <SqsMessage <T> >(response.Messages.Count);
foreach (var message in response.Messages)
{
var properties = new Dictionary <string, string>(message.Attributes.Count + message.MessageAttributes.Count, StringComparer.Ordinal);
foreach (var attribute in message.Attributes)
{
properties[attribute.Key] = attribute.Value;
}
foreach (var msgAttribute in message.MessageAttributes)
{
properties[msgAttribute.Key] = msgAttribute.Value.StringValue;
}
var contentType = ContentTypes.Json;
if (message.MessageAttributes.TryGetValue(KnownProperties.ContentType, out var cta))
{
contentType = cta.StringValue;
}
var messageType = default(string);
if (message.MessageAttributes.TryGetValue(KnownProperties.ValueTypeName, out var vtn))
{
messageType = vtn.StringValue;
}
if (_acknowledgementAction is null)
{
Throw.Exception("Acknowledgement action cannot be null for SQS message");
}
try
{
result.Add(new SqsMessage <T>(_acknowledgementAction)
{
Id = message.MessageId,
Properties = properties,
ReceiptHandle = message.ReceiptHandle,
QueueUrl = _currentConsumerOptions.QueueUrl,
Value = _deserializationProvider.From <T>(message.Body, contentType, messageType ?? string.Empty)
});
}
catch (Exception e)
{
// Swallow deserialization exception to prevent blocking the pipeline and processing of subsequent messages.
_logger.LogError(e, $"Error deserializing message body. {e.Message}. {nameof(message.MessageId)}:{message.MessageId}. {nameof(message.MD5OfBody)}:{message.MD5OfBody}");
}
}
return(result);
}
