KafkaConsumer CreateQueueConsumer(string queue, string consumerId = null, int backOffIncrement = 30, int fullLoadThreshold = 1000, int waitInterval = 1000)
{
CreateTopicIfNotExists(queue);
var queueConsumer = new KafkaConsumer(_zkConnectionString, queue, $"{queue}.consumer", consumerId, backOffIncrement, fullLoadThreshold, waitInterval);
return(queueConsumer);
}
void ReceiveMessages(CancellationTokenSource cancellationTokenSource, OnMessagesReceived onMessagesReceived, KafkaConsumer kafkaConsumer)
{
IEnumerable <Kafka.Client.Messages.Message> messages = null;
#region peek messages that not been consumed since last time
while (!cancellationTokenSource.IsCancellationRequested)
{
try
{
messages = kafkaConsumer.GetMessages(cancellationTokenSource.Token);
foreach (var message in messages)
{
try
{
var kafkaMessage = Encoding.UTF8.GetString(message.Payload).ToJsonObject <KafkaMessage>();
var messageContext = new MessageContext(kafkaMessage, message.PartitionId.Value, message.Offset);
kafkaConsumer.AddMessage(message);
onMessagesReceived(messageContext);
kafkaConsumer.BlockIfFullLoad();
}
catch (OperationCanceledException)
{
return;
}
catch (ThreadAbortException)
{
return;
}
catch (Exception ex)
{
if (message.Payload != null)
{
kafkaConsumer.RemoveMessage(message.PartitionId.Value, message.Offset);
}
_logger.Error(ex.GetBaseException().Message, ex);
}
}
}
catch (OperationCanceledException)
{
return;
}
catch (ThreadAbortException)
{
return;
}
catch (Exception ex)
{
if (!cancellationTokenSource.IsCancellationRequested)
{
Thread.Sleep(1000);
_logger.Error(ex.GetBaseException().Message, ex);
}
}
}
#endregion
}
