Example #1
0
        private static void AddKafkaMessageBus(IServiceCollection services)
        {
            var bootstrapServers = "192.168.111.132:9092,192.168.111.132:9093,192.168.111.132:9094"; // com 虚拟机
                                                                                                     // bootstrapServers = "192.168.72.130:9092,192.168.72.130:9093,192.168.72.130:9094";//home 虚拟机
            var options = new KafkaMessageBusOptions
            {
                ClientMode  = ClientMode.Both,
                TopicPrefix = "KafkaDemo", //项目名称
                //TopicMode = TopicMode.Single,
                ConsumerThreadCount = 4,
                ManualCommitBatch   = 10,
                ProducerConfig      = new ProducerConfig
                {
                    BootstrapServers = bootstrapServers,
                    Acks             = Acks.Leader,
                    Partitioner      = Partitioner.ConsistentRandom
                },
                ConsumerConfig = new ConsumerConfig
                {
                    GroupId                = "demo-messagebus",
                    BootstrapServers       = bootstrapServers,
                    AutoOffsetReset        = AutoOffsetReset.Earliest,
                    EnableAutoCommit       = false, //手动提交,避免数据丢失。但是数据重复问题避免不了,需要业务上做处理
                    AutoCommitIntervalMs   = 5000,  //自动提交偏移量间隔 ,每5秒同步一次,当再均衡时,如果有消费者一直没有poll,会等到所有的消费者都poll之后才再均衡处理
                    CancellationDelayMaxMs = 1000   //poll等待时间,如果自己 consumer.Consume(TimeSpan.FromSeconds(1));写的时间就不用这个配置了
                }
            };



            services.AddKafkaMessageBus(options);
        }
Example #2
0
        public KafkaConsumer(IServiceProvider serviceProvider)
        {
            _serviceProvider = serviceProvider;

            _logger                = serviceProvider.GetService <ILogger <KafkaConsumer <TKey, TValue> > >();
            _kafkaOptions          = serviceProvider.GetService <KafkaMessageBusOptions>();
            CancellationDelayMaxMs = _kafkaOptions.CancellationDelayMaxMs > 0 ? _kafkaOptions.CancellationDelayMaxMs : 100;
        }
        public KafkaProducer(IServiceProvider serviceProvider)
        {
            _serviceProvider = serviceProvider;

            _logger       = serviceProvider.GetService <ILogger <KafkaProducer <TKey, TValue> > >();
            _kafkaOptions = serviceProvider.GetService <KafkaMessageBusOptions>();

            this.CreateProducer();
        }
Example #4
0
        private static void Main(string[] args)
        {
            //已经在项目文件中设置了
            //System.Threading.ThreadPool.SetMinThreads(100, 100);

            //任务系统出错的情况
            TaskScheduler.UnobservedTaskException += TaskScheduler_UnobservedTaskException;

            // 创建Host ,这里可以进一步简化
            var host = new HostBuilder()
                       .ConfigureLogging((context, factory) =>
            {
                factory.SetMinimumLevel(LogLevel.Information);
                factory.AddConsole();
            })
                       .ConfigureServices(service =>
            {
                service.Configure <KafkaMessageBusOptions>(o =>
                {
                });
                service.AddSingleton <IMessageBus>(p =>
                {
                    var options = new KafkaMessageBusOptions
                    {
                        ClientMode           = ClientMode.Both,
                        BootStrapServers     = "127.0.0.1:9092",
                        GroupId              = "FTestGroup",
                        Topic                = "FTestTopic",
                        AutoCommitIntervalMS = 5000,
                        AutoOffSetReset      = "earliest"
                    };
                    return(new KafkaMessageBus(options, p.GetRequiredService <ILoggerFactory>()));
                });

                service.AddHostedService <MessageBusConsumeService>();
                service.AddHostedService <MessageBusProduerService>();
            });

            host.RunConsoleAsync().Wait();
        }
        public static string GetTopic(KafkaMessageBusOptions options, Type type)
        {
            string topicName = null;

            if (TopicCache.TryGetValue(type, out topicName))
            {
                return(topicName);
            }

            if (!string.IsNullOrEmpty(options.Topic))
            {
                topicName = options.Topic;
                topicName = $"{options.TopicPrefix ?? ""}{topicName}";
            }
            else
            {
                topicName = type.Name; //默认等于该类型的名称
                var topicAttr = TopicAttribute.GetTopicAttribute(type);
                if (topicAttr != null && !string.IsNullOrEmpty(topicAttr.Name))
                {
                    topicName = topicAttr.Name;
                }
                else
                {
                    var displayAttr = AttributeUtils.GetAttribute <DisplayAttribute>(type);
                    if (displayAttr != null && !string.IsNullOrEmpty(displayAttr.Name))
                    {
                        topicName = displayAttr.Name;
                    }
                }
                topicName = $"{options.TopicPrefix ?? ""}{topicName}";
            }


            TopicCache.TryAdd(type, topicName);

            return(topicName);
        }