public static void Main(string[] args)
{
using (var producer = new Producer("localhost:9092"))
using (var topic = producer.Topic("test"))
{
var data = Encoding.UTF8.GetBytes("Hello Kafka");
var deliveryReport = topic.Produce(data).Result;
Console.WriteLine($"Produced to Partition: {deliveryReport.Partition}, Offset: {deliveryReport.Offset}");
}
var config = new Config {
GroupId = "test-consumer"
};
using (var consumer = new EventConsumer(config, "localhost:9092"))
{
consumer.OnMessage += (obj, msg) =>
{
var text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
consumer.Subscribe(new List <string> {
"test"
});
consumer.Start();
Console.ReadLine();
}
}
static void Main(string[] args)
{
Console.OutputEncoding = Encoding.UTF8;
Dictionary <string, List <YuanXinLogContext> > dictionary = new Dictionary <string, List <YuanXinLogContext> >();
//配置消费者组
try
{
var config = new Config()
{
GroupId = "example-csharp-consumer"
};
using (var consumer = new EventConsumer(config, ""))
{
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
var myclient = msg.Payload.DeSerialize <YuanXinLogContext>();
var key = myclient.ContextId;
if (dictionary.ContainsKey(key))
{
List <YuanXinLogContext> list;
dictionary.TryGetValue(key, out list);
list.Add(myclient);
dictionary.Remove(key);
dictionary.Add(key, list);
}
else
{
dictionary.Add(key, new List <YuanXinLogContext> {
myclient
});
}
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset}");
};
//订阅一个或者多个Topic
var tops = new List <string>();
tops.Add("LoggerTopic");
consumer.Subscribe(tops);
//启动
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex}");
}
}
/// <summary>
/// 启动消息监听
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="topicNames"></param>
/// <param name="groupId">消费者组</param>
/// <param name="host"></param>
/// <param name="consumerAction">处理该类消息的逻辑类</param>
public void SubscribeAt <T>(List <string> topicNames, string groupId, string host, IEventConsumer <T> consumerAction)
{
var config = new Config()
{
GroupId = groupId
};
var consumer = new EventConsumer(config, $"{host}:9092");
consumer.OnMessage += (obj, msg) => { consumerAction.Consumer(msg); };
consumer.OnConsumerError += (obj, error) => { Log.Write($"消费消息异常:{error.ToString()}", MessageType.Error); };
consumer.Subscribe(topicNames);
consumer.Start();
}
public KafkaConsumer(ILogger logger, List <string> topics, IConfigurationSection kafkaConfig, MessageHandler handler)
{
_logger = logger;
_handler = handler;
var config = KafkaConfigReader.Read(kafkaConfig);
config.EnableAutoCommit = false;
KafkaConfigReader.LogConfig(_logger, config);
config.Logger = OnLog;
_consumer = new EventConsumer(config);
_consumer.Subscribe(topics);
_consumer.OnMessage += OnMessage;
logger.LogInformation("Starting consumer");
_consumer.Start();
}
public async Task TestAsync()
{
// Producer 接受一个或多个 BrokerList
using (Producer producer = new Producer("132.232.4.73:9092"))
//发送到一个名为 testtopic 的Topic，如果没有就会创建一个
using (Topic topic = producer.Topic("testtopic"))
{
//将message转为一个 byte[]
byte[] data = Encoding.UTF8.GetBytes("Hello RdKafka");
DeliveryReport deliveryReport = await topic.Produce(data);
Console.WriteLine($"发送到分区：{deliveryReport.Partition}, Offset 为: {deliveryReport.Offset}");
}
//配置消费者组
var config = new Config()
{
GroupId = "example-csharp-consumer"
};
using (var consumer = new EventConsumer(config, "132.232.4.73:9092"))
{
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
//订阅一个或者多个Topic
consumer.Subscribe(new[] { "testtopic" });
//启动
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
// WIP, not producing useful numbers yet. Assumes one partition.
public static async Task <long> Consume(string broker, string topic)
{
long n = 0;
var topicConfig = new TopicConfig();
topicConfig["auto.offset.reset"] = "smallest";
var config = new Config()
{
GroupId = "benchmark-consumer",
DefaultTopicConfig = topicConfig
};
using (var consumer = new EventConsumer(config, broker))
{
var signal = new SemaphoreSlim(0, 1);
consumer.OnMessage += (obj, msg) =>
{
n += 1;
};
consumer.OnEndReached += (obj, end) =>
{
Console.WriteLine($"End reached");
signal.Release();
};
consumer.Subscribe(new List <string> {
topic
});
consumer.Start();
await signal.WaitAsync();
Console.WriteLine($"Shutting down");
}
return(n);
}
public static void BeginConsumer(string consumername, string topic)
{
//配置消费者组
var config = new Config()
{
GroupId = consumername, StatisticsInterval = new TimeSpan(1000)
};
var consumer = new EventConsumer(config, baseconfig.KafakServer);
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
try
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
};
//订阅一个或者多个Topic
consumer.Subscribe(new List <string>()
{
topic
});
//var message = consumer.Consume(new TimeSpan(1000));
//var msg = message.Value.Message;
//string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
//Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
//启动
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
//Console.ReadLine();
}
protected override void Retrieve(BrokerRouter mq, Func <string, bool> func, bool noAck)
{
var config = new RdKafka.Config()
{
GroupId = Consumer
};
using (var consumer = new EventConsumer(config, CreateMessageQueue().BrokerServer))
{
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
try
{
//msg.TopicPartitionOffset
func(msg.Payload.ToUtf8String());
}
catch (Exception ex)
{
Logger.Write(new LogExceptionEntity(ex));
if (ConfigEntity.ErrorThrowException)
{
throw new MessageQueueException(ex);
}
}
//Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
//订阅一个或者多个Topic
consumer.Subscribe(ConfigEntity.ExtentProperty[TopicNameKey].Split(',').ToList());
//启动
consumer.Start();
//Console.WriteLine("Started consumer, press enter to stop consuming");
//Console.ReadLine();
}
}
public static void Start(string kafkaUrl, string kafkaTopic)
{
Config config = new Config()
{
GroupId = "example-csharp-consumer"
};
config["security.protocol"] = "ssl";
config["ssl.ca.location"] = ConfigurationManager.AppSettings["ssl.ca.location"];
config["ssl.certificate.location"] = ConfigurationManager.AppSettings["ssl.certificate.location"];
config["ssl.key.location"] = ConfigurationManager.AppSettings["ssl.key.location"];
config["ssl.key.password"] = ConfigurationManager.AppSettings["ssl.key.password"];
consumer = new EventConsumer(config, kafkaUrl);
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
consumer.Subscribe(new List <string> {
kafkaTopic
});
consumer.Start();
}
public void SubscribeTopic <T, TH>(Func <TH> handler, string borkerList, string groupID, List <string> topics)
where TH : IProcessMessage <T>
where T : MessageBase
{
var containsKey = _subsManager.HasSubscriptionsForEvent <T>();
if (!containsKey)
{
Task.Run(() =>
{
try
{
if (string.IsNullOrWhiteSpace(borkerList))
{
borkerList = ConnectionBuilder.getBorkerList();
}
bool enableAutoCommit = false;
var config = new Config()
{
GroupId = groupID,
EnableAutoCommit = enableAutoCommit,
StatisticsInterval = TimeSpan.FromSeconds(60)
};
EventConsumer consumer = new EventConsumer(config, borkerList);
consumer.OnMessage += async(obj, msgs) => {
string text = Encoding.UTF8.GetString(msgs.Payload, 0, msgs.Payload.Length);
_logger.LogInformation("Topic: {0} Partition: {1} Offset: {2} {3}", msgs.Topic, msgs.Partition, msgs.Offset, text);
MessageBase mb = new MessageBase()
{
MessageBodyByte = msgs.Payload,
MessageRouter = msgs.Topic
};
await ProcessEvent(mb.MessageRouter, mb);
if (!enableAutoCommit && msgs.Offset % 10 == 0)
{
_logger.LogInformation($"Committing offset");
consumer.Commit(msgs).Wait();
_logger.LogInformation($"Committed offset");
}
};
consumer.OnConsumerError += (obj, errorCode) =>
{
_logger.LogWarning("Consumer Error: {0}", errorCode);
};
consumer.OnEndReached += (obj, end) => {
_logger.LogInformation("Reached end of topic {0} partition {1}, next message will be at offset {2}", end.Topic, end.Partition, end.Offset);
};
consumer.OnError += (obj, error) => {
_logger.LogWarning("Error: {0} {1}", error.ErrorCode, error.Reason);
};
if (enableAutoCommit)
{
consumer.OnOffsetCommit += (obj, commit) => {
if (commit.Error != ErrorCode.NO_ERROR)
{
_logger.LogInformation("Failed to commit offsets: {0}", commit.Error);
}
_logger.LogInformation("Successfully committed offsets: [{0}]", string.Join(", ", commit.Offsets));
};
}
consumer.OnPartitionsAssigned += (obj, partitions) => {
_logger.LogInformation("Assigned partitions: [{0}], member id: {1}", string.Join(", ", partitions), consumer.MemberId);
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (obj, partitions) => {
_logger.LogInformation("Revoked partitions: [{0}]", string.Join(", ", partitions));
consumer.Unassign();
};
consumer.OnStatistics += (obj, json) => {
_logger.LogInformation("Statistics: {0}", json);
};
consumer.Subscribe(topics);
consumer.Start();
_logger.LogInformation("Assigned to: [{0}]", string.Join(", ", consumer.Assignment));
_logger.LogInformation("Subscribed to: [{0}]", string.Join(", ", consumer.Subscription));
_logger.LogInformation("Started consumer, press enter to stop consuming");
while (true)
{
Thread.Sleep(1);
}
// Console.ReadLine();
}
catch (Exception ex)
{
_logger.LogError("SubscribeTopicAsync", ex);
return(ex.Message);
}
});//----------end run
}//-----------end if
_subsManager.AddSubscription <T, TH>(handler);
}//-----------end SubscribeTopic
public static void Run(string brokerList, List <string> topics)
{
bool enableAutoCommit = false;
var config = new Config()
{
GroupId = "advanced-csharp-consumer",
EnableAutoCommit = enableAutoCommit,
StatisticsInterval = TimeSpan.FromSeconds(60)
};
using (var consumer = new EventConsumer(config, brokerList))
{
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
Console.WriteLine("1 Response: Partition {0},Offset {1} : {2}",
msg.Partition, msg.Offset, text);
if (!enableAutoCommit && msg.Offset % 10 == 0)
{
Console.WriteLine("Committing offset");
consumer.Commit(msg).Wait();
Console.WriteLine("Committed offset");
}
};
consumer.OnConsumerError += (obj, errorCode) =>
{
Console.WriteLine($"Consumer Error: {errorCode}");
};
consumer.OnEndReached += (obj, end) =>
{
Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
};
consumer.OnError += (obj, error) =>
{
Console.WriteLine($"Error: {error.ErrorCode} {error.Reason}");
};
if (enableAutoCommit)
{
consumer.OnOffsetCommit += (obj, commit) =>
{
if (commit.Error != ErrorCode.NO_ERROR)
{
Console.WriteLine($"Failed to commit offsets: {commit.Error}");
}
Console.WriteLine($"Successfully committed offsets: [{string.Join(", ", commit.Offsets)}]");
};
}
consumer.OnPartitionsAssigned += (obj, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}");
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (obj, partitions) =>
{
Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]");
consumer.Unassign();
};
consumer.OnStatistics += (obj, json) =>
{
Console.WriteLine($"Statistics: {json}");
};
consumer.Subscribe(topics);
consumer.Start();
Console.WriteLine($"Assigned to: [{string.Join(", ", consumer.Assignment)}]");
Console.WriteLine($"Subscribed to: [{string.Join(", ", consumer.Subscription)}]");
Console.WriteLine($"Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
public static void AddSubscriptions(this EventConsumer consumer, List <string> topicsToAdd)
{
consumer.Subscribe(topicsToAdd);
Logger.Debug("Subscriptions added:" + string.Join(", ", topicsToAdd));
}