static void KafkaLongRunningProcessTest(EmployeeDbContext context, string arg)
{
var topic = "SampleEvent";
var conf = new ConsumerConfig
{
GroupId = "sample-event-consumer-group-2",
BootstrapServers = "localhost:9092",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = true,
EnableAutoOffsetStore = false,
MaxPollIntervalMs = 86400000
//MaxPollIntervalMs = 12000,
//AutoCommitIntervalMs = 3000
};
using (var consumer = new ConsumerBuilder <Ignore, string>(conf).Build())
{
consumer.Subscribe(topic);
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
try
{
ConsumeResult <Ignore, string> result = null;
while (true)
{
try
{
result = consumer.Consume(cts.Token);
Employee employee = Get(context);
Console.WriteLine(
$"Consumed message '{ result.Message.Value }' at: { DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss,fff") }'."
);
// Test 100 seconds, 1 minute 40 seconds - works
// Test 200 seconds, 3 minutes 20 seconds - works
// Test 400 seconds, 6 minutes 40 seconds - Hit error: Application maximum poll interval (300000ms)
// exceeded by 98ms (adjust max.poll.interval.ms for long-running message processing): leaving group
// Resolve by setting MaxPollIntervalMs = 86400000
// Test 800 seconds, 13 minutes 20 seconds - works
// Test 3000 seconds, 50 minutes - works
Thread.Sleep(TimeSpan.FromMilliseconds(30000));
Update(context, employee, arg);
//consumer.Commit(result); // sync commit will failed if consumer leave group
consumer.StoreOffset(result); // async commit with background thread
Console.WriteLine(
$"Commit: { DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss,fff") }"
);
}
catch (ConsumeException ex)
{
Console.WriteLine($"Error occured: { ex.Error.Reason }");
}
catch (TopicPartitionException ex)
{
Console.WriteLine($"Commit error: { ex.Error.Reason }");
}
catch (KafkaException ex)
{
Console.WriteLine($"Commit error: { ex.Error.Reason }");
}
catch (DbUpdateConcurrencyException ex)
{
consumer.StoreOffset(result);
ex.Entries.Single().Reload();
Console.WriteLine($"DbUpdateConcurrencyException error: { ex.Message }");
}
catch (Exception ex)
{
Console.WriteLine($"General error: { ex.Message }");
}
}
}
catch (OperationCanceledException)
{
// Ensure the consumer leaves the group cleanly and final offsets are committed.
consumer.Close();
}
finally
{
consumer.Close();
}
}
}