private static void AddKafkaMessageBus(IServiceCollection services)
{
var bootstrapServers = "192.168.111.132:9092,192.168.111.132:9093,192.168.111.132:9094"; // com 虚拟机
// bootstrapServers = "192.168.72.130:9092,192.168.72.130:9093,192.168.72.130:9094";//home 虚拟机
var options = new KafkaMessageBusOptions
{
ClientMode = ClientMode.Both,
TopicPrefix = "KafkaDemo", //项目名称
//TopicMode = TopicMode.Single,
ConsumerThreadCount = 4,
ManualCommitBatch = 10,
ProducerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
Acks = Acks.Leader,
Partitioner = Partitioner.ConsistentRandom
},
ConsumerConfig = new ConsumerConfig
{
GroupId = "demo-messagebus",
BootstrapServers = bootstrapServers,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false, //手动提交，避免数据丢失。但是数据重复问题避免不了，需要业务上做处理
AutoCommitIntervalMs = 5000, //自动提交偏移量间隔 ，每5秒同步一次,当再均衡时，如果有消费者一直没有poll，会等到所有的消费者都poll之后才再均衡处理
CancellationDelayMaxMs = 1000 //poll等待时间，如果自己 consumer.Consume(TimeSpan.FromSeconds(1));写的时间就不用这个配置了
}
};
services.AddKafkaMessageBus(options);
}
public KafkaConsumer(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
_logger = serviceProvider.GetService <ILogger <KafkaConsumer <TKey, TValue> > >();
_kafkaOptions = serviceProvider.GetService <KafkaMessageBusOptions>();
CancellationDelayMaxMs = _kafkaOptions.CancellationDelayMaxMs > 0 ? _kafkaOptions.CancellationDelayMaxMs : 100;
}
public KafkaProducer(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
_logger = serviceProvider.GetService <ILogger <KafkaProducer <TKey, TValue> > >();
_kafkaOptions = serviceProvider.GetService <KafkaMessageBusOptions>();
this.CreateProducer();
}
private static void Main(string[] args)
{
//已经在项目文件中设置了
//System.Threading.ThreadPool.SetMinThreads(100, 100);
//任务系统出错的情况
TaskScheduler.UnobservedTaskException += TaskScheduler_UnobservedTaskException;
// 创建Host ，这里可以进一步简化
var host = new HostBuilder()
.ConfigureLogging((context, factory) =>
{
factory.SetMinimumLevel(LogLevel.Information);
factory.AddConsole();
})
.ConfigureServices(service =>
{
service.Configure <KafkaMessageBusOptions>(o =>
{
});
service.AddSingleton <IMessageBus>(p =>
{
var options = new KafkaMessageBusOptions
{
ClientMode = ClientMode.Both,
BootStrapServers = "127.0.0.1:9092",
GroupId = "FTestGroup",
Topic = "FTestTopic",
AutoCommitIntervalMS = 5000,
AutoOffSetReset = "earliest"
};
return(new KafkaMessageBus(options, p.GetRequiredService <ILoggerFactory>()));
});
service.AddHostedService <MessageBusConsumeService>();
service.AddHostedService <MessageBusProduerService>();
});
host.RunConsoleAsync().Wait();
}
public static string GetTopic(KafkaMessageBusOptions options, Type type)
{
string topicName = null;
if (TopicCache.TryGetValue(type, out topicName))
{
return(topicName);
}
if (!string.IsNullOrEmpty(options.Topic))
{
topicName = options.Topic;
topicName = $"{options.TopicPrefix ?? ""}{topicName}";
}
else
{
topicName = type.Name; //默认等于该类型的名称
var topicAttr = TopicAttribute.GetTopicAttribute(type);
if (topicAttr != null && !string.IsNullOrEmpty(topicAttr.Name))
{
topicName = topicAttr.Name;
}
else
{
var displayAttr = AttributeUtils.GetAttribute <DisplayAttribute>(type);
if (displayAttr != null && !string.IsNullOrEmpty(displayAttr.Name))
{
topicName = displayAttr.Name;
}
}
topicName = $"{options.TopicPrefix ?? ""}{topicName}";
}
TopicCache.TryAdd(type, topicName);
return(topicName);
}