public KafkaMessageSender(ILoggerFactory loggerFactory, KafkaStreamProviderOptions options)
{
_logger = loggerFactory.CreateLogger <KafkaMessageSender>();
var producerConfig = new Dictionary <string, object>
{
// https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
{ "bootstrap.servers", options.BrokerEndpoints },
{ "api.version.request", true },
{ "queue.buffering.max.ms", options.Producer.QueueBufferingMaxMs },
{ "queue.buffering.max.kbytes", options.Producer.QueueBufferingMaxKbytes },
{ "batch.num.messages", options.Producer.BatchNumMessages },
{ "message.max.bytes", options.Producer.MessageMaxBytes },
#if DEBUG
{ "debug", "msg" },
{ "socket.blocking.max.ms", 1 }, // https://github.com/edenhill/librdkafka/wiki/How-to-decrease-message-latency
#else
{ "log.connection.close", false },
#endif
{
"default.topic.config",
new Dictionary <string, object>
{
{ "message.timeout.ms", 5000 },
{ "request.required.acks", -1 }
}
}
};
_producer = new Producer <string, string>(producerConfig, new StringSerializer(Encoding.UTF8), new StringSerializer(Encoding.UTF8));
_producer.OnLog += OnLog;
_producer.OnError += OnLogError;
_producer.OnStatistics += OnStatistics;
}
private static Dictionary <string, object> CreateConsumerConfig(KafkaStreamProviderOptions kafkaOptions)
=> new Dictionary <string, object>
{
// https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
{ "bootstrap.servers", kafkaOptions.BrokerEndpoints },
{ "api.version.request", true },
{ "group.id", string.IsNullOrEmpty(kafkaOptions.Consumer.ConsumerGroup) ? Guid.NewGuid().ToString() : kafkaOptions.Consumer.ConsumerGroup },
{ "enable.auto.commit", kafkaOptions.Consumer.EnableAutoCommit },
{ "fetch.wait.max.ms", kafkaOptions.Consumer.FetchWaitMaxMs },
{ "fetch.error.backoff.ms", kafkaOptions.Consumer.FetchErrorBackoffMs },
{ "fetch.message.max.bytes", kafkaOptions.Consumer.FetchMessageMaxBytes },
{ "queued.min.messages", kafkaOptions.Consumer.QueuedMinMessages },
#if DEBUG
//{ "debug", "consumer,cgrp,topic,fetch" },
{ "socket.blocking.max.ms", 1 }, // https://github.com/edenhill/librdkafka/wiki/How-to-decrease-message-latency
#else
{ "log.connection.close", false },
#endif
{
"default.topic.config",
new Dictionary <string, object>
{
{ "auto.offset.reset", "beginning" }
}
}
};
public KafkaQueueAdapterReceiver(
string providerName,
QueueId queueId,
KafkaStreamProviderOptions options,
ILoggerFactory loggerFactory)
: base(loggerFactory.CreateLogger <KafkaConsumerWrapper>(), options)
{
Id = queueId ?? throw new ArgumentNullException(nameof(queueId));
_providerName = providerName;
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = loggerFactory.CreateLogger <KafkaQueueAdapterReceiver>();
}
protected KafkaConsumerWrapper(ILogger logger, KafkaStreamProviderOptions options)
{
_logger = logger;
Consumer = new Consumer <string, string>(
CreateConsumerConfig(options),
new StringDeserializer(Encoding.UTF8),
new StringDeserializer(Encoding.UTF8));
Consumer.OnLog += OnLog;
Consumer.OnError += OnError;
Consumer.OnStatistics += OnStatistics;
Consumer.OnConsumeError += OnConsumeError;
}
public KafkaQueueAdapter(
HashRingBasedStreamQueueMapper queueMapper,
KafkaStreamProviderOptions options,
string providerName,
ILoggerFactory loggerFactory)
{
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_options = options ?? throw new ArgumentNullException(nameof(options));
_streamQueueMapper = queueMapper ?? throw new ArgumentNullException(nameof(queueMapper));
Name = providerName;
_loggerFactory = loggerFactory;
_logger = loggerFactory.CreateLogger <KafkaQueueAdapter>();
_producer = new KafkaMessageSender(loggerFactory, options);
_logger.LogInformation("{0} - Created", nameof(KafkaQueueAdapter));
}
public KafkaQueueAdapterFactory(
string name,
KafkaStreamProviderOptions options,
ILoggerFactory loggerFactory)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
_options = options ?? throw new ArgumentNullException(nameof(options));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_logger = _loggerFactory.CreateLogger <KafkaQueueAdapterFactory>();
_providerName = name;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(
new HashRingStreamQueueMapperOptions {
TotalQueueCount = _options.PartitionsCount
},
name);
_adapterCache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions {
CacheSize = options.Cache.CacheSize
}, _providerName, _loggerFactory);
}