public MessageReaderTask <TKey, TValue, TContract> Build(SubscriptionConfiguration subscription)
{
var group = subscription.Group ?? _options.Configuration?.Group;
if (string.IsNullOrEmpty(_options.Server))
{
throw new ArgumentException("Kafka connection string is not defined");
}
var config = new ConsumerConfig(_options.Configuration?.Consumer ?? new())
{
BootstrapServers = _options.Server,
GroupId = group,
};
var consumer = _clientFactory?.CreateConsumer <TKey, TValue>(_options, subscription);
if (consumer is null)
{
throw new ArgumentNullException(nameof(consumer), "Consumer build failure");
}
consumer.Subscribe(subscription.Topic);
return(new MessageReaderTask <TKey, TValue, TContract>(
subscription.Scope.ServiceProvider.GetServices <IMessageInterceptor>(),
subscription.Serializer,
subscription.Logger,
consumer,
subscription.Topic));
}
public static IEnumerable <TopicPartitionOffset> Handler <TKey, TValue>(ILogger logger,
SubscriptionConfiguration subscription,
IConsumer <TKey, TValue> consumer,
List <TopicPartition> partitions)
{
logger.LogInformation("Assignment for {Subscription}({Partitions}): {Assignment}",
string.Join(", ", consumer.Subscription),
string.Join(", ", partitions.Select(x => x.Partition.Value)),
string.Join(", ", consumer.Assignment));
Offset Bias(TopicPartition partition)
{
var range = new Lazy <WatermarkOffsets>(() =>
consumer.QueryWatermarkOffsets(partition, TimeSpan.FromSeconds(5)));
var current = new Lazy <Offset>(() => consumer
.Committed(new[] { partition }, TimeSpan.FromSeconds(5))
.Select(x => x.Offset)
.Concat(new[] { Offset.Unset })
.First()
.Otherwise(range.Value.Low));
return(subscription.Offset switch
{
TopicOffset.Begin => subscription.Bias == 0
? Offset.Beginning
: Math.Clamp(range.Value.Low + subscription.Bias, range.Value.Low, range.Value.High),
TopicOffset.End => subscription.Bias == 0
? Offset.End
: Math.Clamp(range.Value.High + subscription.Bias, range.Value.Low, range.Value.High),
TopicOffset.Stored => subscription.Bias == 0
? current.Value
: Math.Clamp(current.Value + subscription.Bias, range.Value.Low, range.Value.High),
_ => throw new ArgumentOutOfRangeException(nameof(subscription.Offset))
});
}
public IMessageSubscription Subscribe <T>(
string topic,
Func <IMessage <T>, Task> handler,
SubscriptionOptions options = null) where T : class
{
topic = ExpandTemplate(topic);
var group = ExpandTemplate(Options?.Configuration?.Group)?.ToLowerInvariant();
var format = options?.Format ?? TopicFormat.String;
var offset = options?.Offset is var o and not null and not TopicOffset.Unset
? o.Value
: Options?.Configuration?.Offset ?? TopicOffset.Unset;
if (offset == TopicOffset.Unset)
{
offset = TopicOffset.Stored;
}
var bias = options?.Bias ?? Options?.Configuration?.Bias ?? 0;
#if (DEBUG)
if (!string.IsNullOrEmpty(group))
{
group += "-";
}
group += Environment.MachineName;
#endif
using var _ = Logger.BeginScope(new
{
Topic = topic,
Group = group,
Offset = offset,
Bias = bias,
});
try
{
Logger.LogInformation("* Subscribe topic {Topic} from date: {DateOffset} time: {TimeOffset}, offset: {Offset}, group: {Group}, commit: {CommitMode}",
topic, options?.DateOffset, options?.NegativeTimeOffset, offset, group, Options.IsManualCommit() ? "manual" : "auto");
using var scope = _factory.CreateScope();
var subscription = new SubscriptionConfiguration
{
Topic = topic,
DateOffset = options?.DateOffset,
TimeOffset = options?.NegativeTimeOffset ?? TimeSpan.Zero,
Offset = offset,
Bias = bias,
Group = group,
Logger = Logger,
TopicFormat = format,
LogHandler = LogHandler,
Scope = scope,
Serializer = _serializer,
};
var clientFactory = scope.ServiceProvider.GetService <IKafkaClientFactory>();
return(format == TopicFormat.Avro
? new SubscriptionBuilder <string, GenericRecord, T>(Options, clientFactory).Build(subscription)
.Run(handler)
: new SubscriptionBuilder <string, string, T>(Options, clientFactory).Build(subscription)
.Run(handler));
}
catch (Exception e)
{
Logger.LogError(e, "Subscription failed");
throw;
}
}
