The following implementation covers:
- An abstraction over Confluent.Kafka with a predefined TPL based subscription blocks.
- Subscribe in declarative way as well as a regular fluent style.
- Intercept messages.
- Buffering, batching etc.
- An In-memory broker provider for unit and integration testing.
// Get Kafka bootstrap servers from ConnectionString:Kafka options
services.AddKafka(Configuration);To cover different scenarios - subscriptions can be declared in several ways:
- type marked with a [MessageHandler] attribute and any number of methods (subscriptions) marked with [Message] attribute.
- type that implements IMessageHandler interface and any number of methods (subscriptions) marked with [Message] attribute.
- type that implements [MessageHandler] interface and a [HandleAsync(T)] method (subscription) implementation. For multiple subscriptions within a single type - that type should implement multiple interfaces.
Example 1
var subscription = _consumer.Subscribe("topic-name", x => LogAsync(x), new SubscriptionOptions {
Format = TopicFormat.Avro,
Offset = TopicOffset.Begin,
Bias = -1000,
DateOffset = DateTimeOffset.UtcNow - TimeSpan.FromDays(1),
RelativeOffsetMinutes = TimeSpan.FromDays(1)
});Example 2
var subscription = _consumer
.Pipeline("topic-name", new SubscriptionOptions { ... })
.Buffer(100)
.Batch(100, TimeSpan.FromSeconds(5))
.Action(x => LogAsync(x))
.Commit()
.Subscribe();[Message(Topic = "event.currency.rate-{env}", Format = TopicFormat.Avro)]
public class RateNotification
{
public string Currency { get; set; }
public decimal Rate { get; set; }
}- Subscribe all Types marked with [MessageHandler] attribute.
- Message handler and specific subscription on a method marked with [Message] attribute.
// Kafka message handler
[MessageHandler]
public class RateNotificationMessageHandler
{
// class with proper DI support.
// with message wrapper
[Message] public Task Handler(IMessage<RateNotification> message) { ... };
// or handle payload directly
[Message] public Task Handler(RateNotification message) { ... };
}- Subscribe all Types implementing [IMessageHandler] interface.
- Message handler and specific subscription on a method marked with [Message] attribute.
// Kafka message handler
public class RateNotificationMessageHandler : IMessageHandler
{
// class with proper DI support.
// with message wrapper
[Message] public Task Handler(IMessage<RateNotification> message) { ... }
// or handle payload directly
[Message] public Task Handler(RateNotification message) { ... };
}- Subscribe all Types implementing [IMessageHandler] interface.
- Message handler and specific subscription on a [Handle] method that implements IMessageHandler.
// with message wrapper
public class RateNotificationMessageHandler : IMessageHandler<IMessage<RateNotification>>
{
// class with proper DI support.
public Task HandleAsync(IMessage<RateNotification> message) { ... }
}
// or handle payload directly
public class RateNotificationMessageHandler : IMessageHandler<RateNotification>
{
// class with proper DI support.
public Task HandleAsync(RateNotification message) { ... }
}// Kafka message handler
public class WithdrawNotificationMessageHandler : IMessageHandler
{
// class with proper DI support.
// Inplace topic subscription definition and a backing consumption buffer
[Message(Topic = "withdraw_event-{env}", Format = TopicFormat.Avro, Offset = TopicOffset.Begin))]
public Task Handler(IMessage<WithdrawNotification> message)
{
Console.WriteLine($"Withdraw {message.Value.Amount} {message.Value.Currency}");
return Task.CompletedTask;
}
}[MessageHandler]
public class RateNotificationHandler
{
// required
[Message]
// buffer messages
[Buffer(Size = 100)]
// use constant values
[Batch(Size = 190, Time = 5000)]
//commit after handler finished
[Commit]
// Parameter of type IEnumerable<IMessage<RateNotification>> is also supported
public Task Handler(IMessageEnumerable<RateNotification> messages)
{
Console.WriteLine($"Received batch with size {messages.Count}");
return Task.CompletedTask;
}
}public class MyInterceptor : IMessageInterceptor
{
public Task ConsumeAsync(IMessage<object> message, Exception exception);
{
Console.WriteLine($"{message.Topic} processed. Exception: {exception}");
return Task.CompletedTask;
}
public Task ProduceAsync(string topic, object key, object message, Exception exception)
{
Console.WriteLine($"{message.Topic} produced. Exception: {exception}");
return Task.CompletedTask;
}
}
services
.AddKafka(Configuration)
.AddInterceptor(new MyInterceptor())
// or
.AddInterceptor(x => new MyInterceptor())
// or
.AddInterceptor(typeof(MyInterceptor))
// or
.AddInterceptor<MyInterceptor>();public void ConfigureServices(IServiceCollection services)
{
services
.AddKafka(_config)
.UseInMemoryBroker();
}Implemented as a MetricsInterceptor.
services
.AddKafka(Configuration)
.AddMetrics();{
"Kafka": {
"Group": "consumer-group-name",
"Producer": {
"linger.ms": 5,
"socket.timeout.ms": 15000,
"message.send.max.retries": 10,
"message.timeout.ms": 200000
},
"Consumer": {
"socket.timeout.ms": 15000,
"enable.auto.commit": false
}
},
"ConnectionStrings": {
"Kafka": "192.168.0.1:9092,192.168.0.2:9092",
"SchemaRegistry": "http://192.168.0.1:8084"
}
}