public KafkaConsumerStream(IKafkaMessageStream <Client.Messages.Message> stream)
{
_tokenSource = new CancellationTokenSource();
_stream = stream.GetCancellable(_tokenSource.Token);
_thread = new Thread(RunConsumer);
}
private Task StartConsumer(
IKafkaMessageStream <Client.Messages.Message> stream,
Action <IEnumerable <ConsumedMessage> > dataSubscriber,
Action <Exception> errorSubscriber)
{
var completionSource = new TaskCompletionSource <bool>();
var thread = new Thread(() =>
{
Logger.Info($"Starting consumer thread {Thread.CurrentThread.ManagedThreadId}");
while (_consumer != null)
{
var tokenSource = new CancellationTokenSource(_batchTimeoutMs);
try
{
var cancellable = stream.GetCancellable(tokenSource.Token);
var messages = cancellable
.Select(message => message.AsConsumedMessage())
.ToArray();
Logger.Debug($"Received {messages.Length} messages.");
if (messages.Length == 0)
{
continue;
}
for (var i = 0; i < messages.Length; i += _maxBatchSize)
{
if (_consumer == null)
{
break;
}
var taken = messages.Skip(i).Take(_maxBatchSize).ToArray();
Logger.Debug($"Skip {i}, take {_maxBatchSize}, get {taken.Length}.");
dataSubscriber(taken);
var map = taken
.GroupBy(m => m.Partition)
.ToDictionary(kvp => kvp.Key, kvp => kvp.Max(m => m.Offset + 1));
foreach (var kvp in map)
{
Commit(kvp.Key, kvp.Value);
}
}
}
catch (Exception ex)
{
Logger.Error($"Exception in consumer {_group} for {_topic}. Restarting...", ex);
errorSubscriber?.Invoke(ex);
Restart();
break;
}
finally
{
tokenSource.Dispose();
}
}
Logger.Info($"Ending consumer thread {Thread.CurrentThread.ManagedThreadId}");
completionSource.SetResult(true);
});
_threadList.Add(thread);
thread.Start();
return(completionSource.Task);
}
