public static void Main(string[] args) { //string brokerList = "138.91.140.244";//d4 string brokerList = "40.118.249.252";//b2 //string brokerList = "localhost"; List <String> topicList = new List <string>(); topicList.Add("testBench24"); var topics = topicList; var config = new Config() { GroupId = "simple-csharp-consumer" }; DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); List <Double> latencies = new List <Double>(); using (var consumer = new EventConsumer(config, brokerList)) { consumer.OnMessage += (obj, msg) => { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); //Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); String receivedMessage = text; String[] substrings = receivedMessage.Split('_'); String id = substrings[0]; String sentTimestamp = substrings[1]; TimeSpan diff = DateTime.Now.ToUniversalTime() - origin; String receivedTimestamp = String.Format(Math.Floor(diff.TotalMilliseconds).ToString()); String finalMessage = String.Format(id + "_" + sentTimestamp + "_" + receivedTimestamp); //Console.WriteLine("Payload on Consumer side == " + finalMessage); Double latency = Convert.ToDouble(receivedTimestamp) - Convert.ToDouble(sentTimestamp); //Console.WriteLine("Payload on Consumer side == " + finalMessage + ", latency == " + latency); latencies.Add(latency); //printing only on the nearest 100th device, since frequent reporting adds to latency if (latencies.Count == 1000000) { PrintLatencyReport(latencies); } //Console.WriteLine(""); //Console.WriteLine(latencies.Count); }; consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topics.First(), 0, 5) }); consumer.Start(); Console.WriteLine("Started consumer, press enter to stop consuming"); //Console.ReadLine(); //PrintLatencyReport(latencies); Console.ReadLine(); } }
public KafkaConsumer(string brokerlist, string topic) { var config = new Config { GroupId = "group" }; _consumer = new EventConsumer(config, brokerlist); _topicName = topic; _i = 0; _consumer.OnMessage += (sender, message) => { if (message.Payload.Length <= 0) { return; } _i++; if (_i % 5000 == 0) { var msg = common.Message.Deserialize(message.Payload); Console.WriteLine($"Topic {_topicName} received msg: {msg.Title} - {msg.Body} - {msg.DeviceId}"); } }; //_consumer.Subscribe(new List<string>{topic}); _consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, Offset.End) }); _consumer.Start(); }
public static void Main(string[] args) { string brokerList = "localhost"; var topics = args.Skip(1).ToList(); var config = new Config() { GroupId = "simple-csharp-consumer" }; using (var consumer = new EventConsumer(config, brokerList)) { consumer.OnMessage += (obj, msg) => { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); }; consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset("topicName", 0, 5) }); consumer.Start(); Console.WriteLine("Started consumer, press enter to stop consuming"); Console.ReadLine(); } }
public void Start() { var config = new Config() { GroupId = "simple-csharp-consumer" }; using (var consumer = new EventConsumer(config, "")) { consumer.OnMessage += (obj, msg) => { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); }; consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topics.First(), 0, 5) }); consumer.Start(); Console.WriteLine("Started consumer, press enter to stop consuming"); Console.ReadLine(); } Console.WriteLine("Starting..."); }
public static void Main(string[] args) { using (var producer = new Producer("localhost:9092")) using (var topic = producer.Topic("test")) { var data = Encoding.UTF8.GetBytes("Hello Kafka"); var deliveryReport = topic.Produce(data).Result; Console.WriteLine($"Produced to Partition: {deliveryReport.Partition}, Offset: {deliveryReport.Offset}"); } var config = new Config { GroupId = "test-consumer" }; using (var consumer = new EventConsumer(config, "localhost:9092")) { consumer.OnMessage += (obj, msg) => { var text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); }; consumer.Subscribe(new List <string> { "test" }); consumer.Start(); Console.ReadLine(); } }
public static void Main(string[] args) { //string brokerList = "40.118.249.252"; string brokerList = "138.91.140.244"; //string brokerList = "localhost"; List <String> topicList = new List <string>(); topicList.Add("testLikeWin"); var topics = topicList; var config = new Config() { GroupId = "simple-csharp-consumer" }; using (var consumer = new EventConsumer(config, brokerList)) { consumer.OnMessage += (obj, msg) => { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); }; consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topics.First(), 0, 5) }); consumer.Start(); Console.WriteLine("Started consumer, press enter to stop consuming"); Console.ReadLine(); } }
static void Main(string[] args) { Console.OutputEncoding = Encoding.UTF8; Dictionary <string, List <YuanXinLogContext> > dictionary = new Dictionary <string, List <YuanXinLogContext> >(); //配置消费者组 try { var config = new Config() { GroupId = "example-csharp-consumer" }; using (var consumer = new EventConsumer(config, "")) { //注册一个事件 consumer.OnMessage += (obj, msg) => { var myclient = msg.Payload.DeSerialize <YuanXinLogContext>(); var key = myclient.ContextId; if (dictionary.ContainsKey(key)) { List <YuanXinLogContext> list; dictionary.TryGetValue(key, out list); list.Add(myclient); dictionary.Remove(key); dictionary.Add(key, list); } else { dictionary.Add(key, new List <YuanXinLogContext> { myclient }); } Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset}"); }; //订阅一个或者多个Topic var tops = new List <string>(); tops.Add("LoggerTopic"); consumer.Subscribe(tops); //启动 consumer.Start(); Console.WriteLine("Started consumer, press enter to stop consuming"); Console.ReadLine(); } } catch (Exception ex) { Console.WriteLine($"{ex}"); } }
/// <summary> /// 启动消息监听 /// </summary> /// <typeparam name="T"></typeparam> /// <param name="topicNames"></param> /// <param name="groupId">消费者组</param> /// <param name="host"></param> /// <param name="consumerAction">处理该类消息的逻辑类</param> public void SubscribeAt <T>(List <string> topicNames, string groupId, string host, IEventConsumer <T> consumerAction) { var config = new Config() { GroupId = groupId }; var consumer = new EventConsumer(config, $"{host}:9092"); consumer.OnMessage += (obj, msg) => { consumerAction.Consumer(msg); }; consumer.OnConsumerError += (obj, error) => { Log.Write($"消费消息异常:{error.ToString()}", MessageType.Error); }; consumer.Subscribe(topicNames); consumer.Start(); }
public void StartConsumer() { if (!ConsumerStarted) { lock (o) { if (!ConsumerStarted) { ConsumerStarted = true; consumer.Start(); } } } }
public KafkaConsumer(ILogger logger, List <string> topics, IConfigurationSection kafkaConfig, MessageHandler handler) { _logger = logger; _handler = handler; var config = KafkaConfigReader.Read(kafkaConfig); config.EnableAutoCommit = false; KafkaConfigReader.LogConfig(_logger, config); config.Logger = OnLog; _consumer = new EventConsumer(config); _consumer.Subscribe(topics); _consumer.OnMessage += OnMessage; logger.LogInformation("Starting consumer"); _consumer.Start(); }
public async Task Test1() { //Larva.MessageProcess.LoggerManager.SetLoggerProvider(new Log4NetLoggerProvider()); var consumer1 = new EventConsumer(); consumer1.Initialize(new ConsumerSettings { AmqpUri = new Uri("amqp://*****:*****@localhost/test") }, "MessageProcess_EventTopic", 4, false, 1, null, typeof(DomainEventTests).Assembly); consumer1.Start(); var consumer2 = new EventConsumer(); consumer2.Initialize(new ConsumerSettings { AmqpUri = new Uri("amqp://*****:*****@localhost/test"), GroupName = "Subscriber2" }, "MessageProcess_EventTopic", 4, true, 1, null, typeof(DomainEventTests).Assembly); consumer2.Start(); var eventBus = new EventBus(); eventBus.Initialize(new ProducerSettings { AmqpUri = new Uri("amqp://*****:*****@localhost/test") }, "MessageProcess_EventTopic"); eventBus.Start(); for (var i = 1; i <= 5; i++) { for (var j = 1; j <= 5; j++) { await eventBus.PublishAsync(new DomainEvent1($"Test{i}")); } } Thread.Sleep(1000); eventBus.Shutdown(); Thread.Sleep(10000); consumer1.Shutdown(); consumer2.Shutdown(); }
public async Task TestAsync() { // Producer 接受一个或多个 BrokerList using (Producer producer = new Producer("132.232.4.73:9092")) //发送到一个名为 testtopic 的Topic,如果没有就会创建一个 using (Topic topic = producer.Topic("testtopic")) { //将message转为一个 byte[] byte[] data = Encoding.UTF8.GetBytes("Hello RdKafka"); DeliveryReport deliveryReport = await topic.Produce(data); Console.WriteLine($"发送到分区:{deliveryReport.Partition}, Offset 为: {deliveryReport.Offset}"); } //配置消费者组 var config = new Config() { GroupId = "example-csharp-consumer" }; using (var consumer = new EventConsumer(config, "132.232.4.73:9092")) { //注册一个事件 consumer.OnMessage += (obj, msg) => { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); }; //订阅一个或者多个Topic consumer.Subscribe(new[] { "testtopic" }); //启动 consumer.Start(); Console.WriteLine("Started consumer, press enter to stop consuming"); Console.ReadLine(); } }
// WIP, not producing useful numbers yet. Assumes one partition. public static async Task <long> Consume(string broker, string topic) { long n = 0; var topicConfig = new TopicConfig(); topicConfig["auto.offset.reset"] = "smallest"; var config = new Config() { GroupId = "benchmark-consumer", DefaultTopicConfig = topicConfig }; using (var consumer = new EventConsumer(config, broker)) { var signal = new SemaphoreSlim(0, 1); consumer.OnMessage += (obj, msg) => { n += 1; }; consumer.OnEndReached += (obj, end) => { Console.WriteLine($"End reached"); signal.Release(); }; consumer.Subscribe(new List <string> { topic }); consumer.Start(); await signal.WaitAsync(); Console.WriteLine($"Shutting down"); } return(n); }
protected override void Retrieve(BrokerRouter mq, Func <string, bool> func, bool noAck) { var config = new RdKafka.Config() { GroupId = Consumer }; using (var consumer = new EventConsumer(config, CreateMessageQueue().BrokerServer)) { //注册一个事件 consumer.OnMessage += (obj, msg) => { try { //msg.TopicPartitionOffset func(msg.Payload.ToUtf8String()); } catch (Exception ex) { Logger.Write(new LogExceptionEntity(ex)); if (ConfigEntity.ErrorThrowException) { throw new MessageQueueException(ex); } } //Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); }; //订阅一个或者多个Topic consumer.Subscribe(ConfigEntity.ExtentProperty[TopicNameKey].Split(',').ToList()); //启动 consumer.Start(); //Console.WriteLine("Started consumer, press enter to stop consuming"); //Console.ReadLine(); } }
public static void BeginConsumer(string consumername, string topic) { //配置消费者组 var config = new Config() { GroupId = consumername, StatisticsInterval = new TimeSpan(1000) }; var consumer = new EventConsumer(config, baseconfig.KafakServer); //注册一个事件 consumer.OnMessage += (obj, msg) => { try { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); } catch (Exception ex) { Console.WriteLine(ex.Message); } }; //订阅一个或者多个Topic consumer.Subscribe(new List <string>() { topic }); //var message = consumer.Consume(new TimeSpan(1000)); //var msg = message.Value.Message; //string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); //Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); //启动 consumer.Start(); Console.WriteLine("Started consumer, press enter to stop consuming"); //Console.ReadLine(); }
public static void Start(string kafkaUrl, string kafkaTopic) { Config config = new Config() { GroupId = "example-csharp-consumer" }; config["security.protocol"] = "ssl"; config["ssl.ca.location"] = ConfigurationManager.AppSettings["ssl.ca.location"]; config["ssl.certificate.location"] = ConfigurationManager.AppSettings["ssl.certificate.location"]; config["ssl.key.location"] = ConfigurationManager.AppSettings["ssl.key.location"]; config["ssl.key.password"] = ConfigurationManager.AppSettings["ssl.key.password"]; consumer = new EventConsumer(config, kafkaUrl); consumer.OnMessage += (obj, msg) => { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); }; consumer.Subscribe(new List <string> { kafkaTopic }); consumer.Start(); }
public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, EventConsumer eventConsumer) { eventConsumer.Start(); app.UseOwin(o => o.UseNancy()); }
public static void Run(string brokerList, List <string> topics) { bool enableAutoCommit = false; var config = new Config() { GroupId = "advanced-csharp-consumer", EnableAutoCommit = enableAutoCommit, StatisticsInterval = TimeSpan.FromSeconds(60) }; using (var consumer = new EventConsumer(config, brokerList)) { consumer.OnMessage += (obj, msg) => { string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length); Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}"); Console.WriteLine("1 Response: Partition {0},Offset {1} : {2}", msg.Partition, msg.Offset, text); if (!enableAutoCommit && msg.Offset % 10 == 0) { Console.WriteLine("Committing offset"); consumer.Commit(msg).Wait(); Console.WriteLine("Committed offset"); } }; consumer.OnConsumerError += (obj, errorCode) => { Console.WriteLine($"Consumer Error: {errorCode}"); }; consumer.OnEndReached += (obj, end) => { Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}"); }; consumer.OnError += (obj, error) => { Console.WriteLine($"Error: {error.ErrorCode} {error.Reason}"); }; if (enableAutoCommit) { consumer.OnOffsetCommit += (obj, commit) => { if (commit.Error != ErrorCode.NO_ERROR) { Console.WriteLine($"Failed to commit offsets: {commit.Error}"); } Console.WriteLine($"Successfully committed offsets: [{string.Join(", ", commit.Offsets)}]"); }; } consumer.OnPartitionsAssigned += (obj, partitions) => { Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}"); consumer.Assign(partitions); }; consumer.OnPartitionsRevoked += (obj, partitions) => { Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]"); consumer.Unassign(); }; consumer.OnStatistics += (obj, json) => { Console.WriteLine($"Statistics: {json}"); }; consumer.Subscribe(topics); consumer.Start(); Console.WriteLine($"Assigned to: [{string.Join(", ", consumer.Assignment)}]"); Console.WriteLine($"Subscribed to: [{string.Join(", ", consumer.Subscription)}]"); Console.WriteLine($"Started consumer, press enter to stop consuming"); Console.ReadLine(); } }
public void SubscribeTopic <T, TH>(Func <TH> handler, string borkerList, string groupID, List <string> topics) where TH : IProcessMessage <T> where T : MessageBase { var containsKey = _subsManager.HasSubscriptionsForEvent <T>(); if (!containsKey) { Task.Run(() => { try { if (string.IsNullOrWhiteSpace(borkerList)) { borkerList = ConnectionBuilder.getBorkerList(); } bool enableAutoCommit = false; var config = new Config() { GroupId = groupID, EnableAutoCommit = enableAutoCommit, StatisticsInterval = TimeSpan.FromSeconds(60) }; EventConsumer consumer = new EventConsumer(config, borkerList); consumer.OnMessage += async(obj, msgs) => { string text = Encoding.UTF8.GetString(msgs.Payload, 0, msgs.Payload.Length); _logger.LogInformation("Topic: {0} Partition: {1} Offset: {2} {3}", msgs.Topic, msgs.Partition, msgs.Offset, text); MessageBase mb = new MessageBase() { MessageBodyByte = msgs.Payload, MessageRouter = msgs.Topic }; await ProcessEvent(mb.MessageRouter, mb); if (!enableAutoCommit && msgs.Offset % 10 == 0) { _logger.LogInformation($"Committing offset"); consumer.Commit(msgs).Wait(); _logger.LogInformation($"Committed offset"); } }; consumer.OnConsumerError += (obj, errorCode) => { _logger.LogWarning("Consumer Error: {0}", errorCode); }; consumer.OnEndReached += (obj, end) => { _logger.LogInformation("Reached end of topic {0} partition {1}, next message will be at offset {2}", end.Topic, end.Partition, end.Offset); }; consumer.OnError += (obj, error) => { _logger.LogWarning("Error: {0} {1}", error.ErrorCode, error.Reason); }; if (enableAutoCommit) { consumer.OnOffsetCommit += (obj, commit) => { if (commit.Error != ErrorCode.NO_ERROR) { _logger.LogInformation("Failed to commit offsets: {0}", commit.Error); } _logger.LogInformation("Successfully committed offsets: [{0}]", string.Join(", ", commit.Offsets)); }; } consumer.OnPartitionsAssigned += (obj, partitions) => { _logger.LogInformation("Assigned partitions: [{0}], member id: {1}", string.Join(", ", partitions), consumer.MemberId); consumer.Assign(partitions); }; consumer.OnPartitionsRevoked += (obj, partitions) => { _logger.LogInformation("Revoked partitions: [{0}]", string.Join(", ", partitions)); consumer.Unassign(); }; consumer.OnStatistics += (obj, json) => { _logger.LogInformation("Statistics: {0}", json); }; consumer.Subscribe(topics); consumer.Start(); _logger.LogInformation("Assigned to: [{0}]", string.Join(", ", consumer.Assignment)); _logger.LogInformation("Subscribed to: [{0}]", string.Join(", ", consumer.Subscription)); _logger.LogInformation("Started consumer, press enter to stop consuming"); while (true) { Thread.Sleep(1); } // Console.ReadLine(); } catch (Exception ex) { _logger.LogError("SubscribeTopicAsync", ex); return(ex.Message); } });//----------end run }//-----------end if _subsManager.AddSubscription <T, TH>(handler); }//-----------end SubscribeTopic