static void KafkaLongRunningProcessTest(EmployeeDbContext context, string arg) { var topic = "SampleEvent"; var conf = new ConsumerConfig { GroupId = "sample-event-consumer-group-2", BootstrapServers = "localhost:9092", AutoOffsetReset = AutoOffsetReset.Earliest, EnableAutoCommit = true, EnableAutoOffsetStore = false, MaxPollIntervalMs = 86400000 //MaxPollIntervalMs = 12000, //AutoCommitIntervalMs = 3000 }; using (var consumer = new ConsumerBuilder <Ignore, string>(conf).Build()) { consumer.Subscribe(topic); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; try { ConsumeResult <Ignore, string> result = null; while (true) { try { result = consumer.Consume(cts.Token); Employee employee = Get(context); Console.WriteLine( $"Consumed message '{ result.Message.Value }' at: { DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss,fff") }'." ); // Test 100 seconds, 1 minute 40 seconds - works // Test 200 seconds, 3 minutes 20 seconds - works // Test 400 seconds, 6 minutes 40 seconds - Hit error: Application maximum poll interval (300000ms) // exceeded by 98ms (adjust max.poll.interval.ms for long-running message processing): leaving group // Resolve by setting MaxPollIntervalMs = 86400000 // Test 800 seconds, 13 minutes 20 seconds - works // Test 3000 seconds, 50 minutes - works Thread.Sleep(TimeSpan.FromMilliseconds(30000)); Update(context, employee, arg); //consumer.Commit(result); // sync commit will failed if consumer leave group consumer.StoreOffset(result); // async commit with background thread Console.WriteLine( $"Commit: { DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss,fff") }" ); } catch (ConsumeException ex) { Console.WriteLine($"Error occured: { ex.Error.Reason }"); } catch (TopicPartitionException ex) { Console.WriteLine($"Commit error: { ex.Error.Reason }"); } catch (KafkaException ex) { Console.WriteLine($"Commit error: { ex.Error.Reason }"); } catch (DbUpdateConcurrencyException ex) { consumer.StoreOffset(result); ex.Entries.Single().Reload(); Console.WriteLine($"DbUpdateConcurrencyException error: { ex.Message }"); } catch (Exception ex) { Console.WriteLine($"General error: { ex.Message }"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. consumer.Close(); } finally { consumer.Close(); } } }