private void CreateBackgroundTask()
{
consumer = this.consumerBuilder.Build();
this.consumerManager.AddOrUpdate(new MessageConsumer(this, this.workerPool, this.configuration, this.logHandler));
consumer.Subscribe(this.configuration.Topics);
this.backgroundTask = Task.Factory.StartNew(
async() =>
{
using (consumer)
{
while (!this.stopCancellationTokenSource.Token.IsCancellationRequested)
{
try
{
var message = consumer.Consume(this.stopCancellationTokenSource.Token);
var headers = new MessageHeaders();
foreach (var header in message.Message.Headers)
{
headers.Add(header.Key, header.GetValueBytes());
}
var intermediateMessage = new IntermediateMessage(headers, message.Message.Value);
intermediateMessage.Topic = message.Topic;
intermediateMessage.Partition = message.Partition;
intermediateMessage.Offset = message.Offset;
await this.workerPool.EnqueueAsync(intermediateMessage, this.stopCancellationTokenSource.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Ignores the exception
}
catch (KafkaException ex) when(ex.Error.IsFatal)
{
this.logHandler.Error("Kafka fatal error occurred. Trying to restart in 5 seconds", ex, null);
await this.workerPool.StopAsync().ConfigureAwait(false);
_ = Task
.Delay(5000, this.stopCancellationTokenSource.Token)
.ContinueWith(t => this.CreateBackgroundTask());
break;
}
catch (Exception ex)
{
this.logHandler.Warning("Error consuming message from Kafka", ex);
}
}
consumer.Close();
}
},
CancellationToken.None,
TaskCreationOptions.LongRunning,
TaskScheduler.Default);
}
public async Task EnqueueAsync(IntermediateMessage message, CancellationToken stopCancellationToken = default)
{
var worker = (IConsumerWorker)await this.distributionStrategy
.GetWorkerAsync(Encoding.UTF8.GetBytes(message.Partition.ToString()), stopCancellationToken)
.ConfigureAwait(false);
if (worker == null)
{
return;
}
this.offsetManager.AddOffset(message.TopicPartitionOffset);
await worker.EnqueueAsync(message, stopCancellationToken).ConfigureAwait(false);
}
//private ConsumerSetting configuration;
public MessageContextConsumer(
IConsumerClient consumerClient,
IOffsetManager offsetManager,
IntermediateMessage kafkaResult,
CancellationToken workerStopped)
{
//this.configuration = configuration;
if (consumerClient != null)
{
this.Name = consumerClient.ConsumerName;
}
this.WorkerStopped = workerStopped;
this.consumerClient = consumerClient;
this.offsetManager = offsetManager;
this.kafkaResult = kafkaResult;
}
public ValueTask EnqueueAsync(
IntermediateMessage message,
CancellationToken stopCancellationToken = default)
{
return(this.messagesBuffer.Writer.WriteAsync(message, stopCancellationToken));
}