public KafkaMessageSender(ILoggerFactory loggerFactory, KafkaStreamProviderOptions options) { _logger = loggerFactory.CreateLogger <KafkaMessageSender>(); var producerConfig = new Dictionary <string, object> { // https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md { "bootstrap.servers", options.BrokerEndpoints }, { "api.version.request", true }, { "queue.buffering.max.ms", options.Producer.QueueBufferingMaxMs }, { "queue.buffering.max.kbytes", options.Producer.QueueBufferingMaxKbytes }, { "batch.num.messages", options.Producer.BatchNumMessages }, { "message.max.bytes", options.Producer.MessageMaxBytes }, #if DEBUG { "debug", "msg" }, { "socket.blocking.max.ms", 1 }, // https://github.com/edenhill/librdkafka/wiki/How-to-decrease-message-latency #else { "log.connection.close", false }, #endif { "default.topic.config", new Dictionary <string, object> { { "message.timeout.ms", 5000 }, { "request.required.acks", -1 } } } }; _producer = new Producer <string, string>(producerConfig, new StringSerializer(Encoding.UTF8), new StringSerializer(Encoding.UTF8)); _producer.OnLog += OnLog; _producer.OnError += OnLogError; _producer.OnStatistics += OnStatistics; }
private static Dictionary <string, object> CreateConsumerConfig(KafkaStreamProviderOptions kafkaOptions) => new Dictionary <string, object> { // https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md { "bootstrap.servers", kafkaOptions.BrokerEndpoints }, { "api.version.request", true }, { "group.id", string.IsNullOrEmpty(kafkaOptions.Consumer.ConsumerGroup) ? Guid.NewGuid().ToString() : kafkaOptions.Consumer.ConsumerGroup }, { "enable.auto.commit", kafkaOptions.Consumer.EnableAutoCommit }, { "fetch.wait.max.ms", kafkaOptions.Consumer.FetchWaitMaxMs }, { "fetch.error.backoff.ms", kafkaOptions.Consumer.FetchErrorBackoffMs }, { "fetch.message.max.bytes", kafkaOptions.Consumer.FetchMessageMaxBytes }, { "queued.min.messages", kafkaOptions.Consumer.QueuedMinMessages }, #if DEBUG //{ "debug", "consumer,cgrp,topic,fetch" }, { "socket.blocking.max.ms", 1 }, // https://github.com/edenhill/librdkafka/wiki/How-to-decrease-message-latency #else { "log.connection.close", false }, #endif { "default.topic.config", new Dictionary <string, object> { { "auto.offset.reset", "beginning" } } } };
public KafkaQueueAdapterReceiver( string providerName, QueueId queueId, KafkaStreamProviderOptions options, ILoggerFactory loggerFactory) : base(loggerFactory.CreateLogger <KafkaConsumerWrapper>(), options) { Id = queueId ?? throw new ArgumentNullException(nameof(queueId)); _providerName = providerName; _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = loggerFactory.CreateLogger <KafkaQueueAdapterReceiver>(); }
protected KafkaConsumerWrapper(ILogger logger, KafkaStreamProviderOptions options) { _logger = logger; Consumer = new Consumer <string, string>( CreateConsumerConfig(options), new StringDeserializer(Encoding.UTF8), new StringDeserializer(Encoding.UTF8)); Consumer.OnLog += OnLog; Consumer.OnError += OnError; Consumer.OnStatistics += OnStatistics; Consumer.OnConsumeError += OnConsumeError; }
public KafkaQueueAdapter( HashRingBasedStreamQueueMapper queueMapper, KafkaStreamProviderOptions options, string providerName, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } _options = options ?? throw new ArgumentNullException(nameof(options)); _streamQueueMapper = queueMapper ?? throw new ArgumentNullException(nameof(queueMapper)); Name = providerName; _loggerFactory = loggerFactory; _logger = loggerFactory.CreateLogger <KafkaQueueAdapter>(); _producer = new KafkaMessageSender(loggerFactory, options); _logger.LogInformation("{0} - Created", nameof(KafkaQueueAdapter)); }
public KafkaQueueAdapterFactory( string name, KafkaStreamProviderOptions options, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(name)) { throw new ArgumentNullException(nameof(name)); } _options = options ?? throw new ArgumentNullException(nameof(options)); _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); _logger = _loggerFactory.CreateLogger <KafkaQueueAdapterFactory>(); _providerName = name; _streamQueueMapper = new HashRingBasedStreamQueueMapper( new HashRingStreamQueueMapperOptions { TotalQueueCount = _options.PartitionsCount }, name); _adapterCache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions { CacheSize = options.Cache.CacheSize }, _providerName, _loggerFactory); }