public static void Main(string[] args)
{
//string brokerList = "138.91.140.244";//d4
string brokerList = "40.118.249.252";//b2
//string brokerList = "localhost";
List <String> topicList = new List <string>();
topicList.Add("testBench24");
var topics = topicList;
var config = new Config()
{
GroupId = "simple-csharp-consumer"
};
DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
List <Double> latencies = new List <Double>();
using (var consumer = new EventConsumer(config, brokerList))
{
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
//Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
String receivedMessage = text;
String[] substrings = receivedMessage.Split('_');
String id = substrings[0];
String sentTimestamp = substrings[1];
TimeSpan diff = DateTime.Now.ToUniversalTime() - origin;
String receivedTimestamp = String.Format(Math.Floor(diff.TotalMilliseconds).ToString());
String finalMessage = String.Format(id + "_" + sentTimestamp + "_" + receivedTimestamp);
//Console.WriteLine("Payload on Consumer side == " + finalMessage);
Double latency = Convert.ToDouble(receivedTimestamp) - Convert.ToDouble(sentTimestamp);
//Console.WriteLine("Payload on Consumer side == " + finalMessage + ", latency == " + latency);
latencies.Add(latency);
//printing only on the nearest 100th device, since frequent reporting adds to latency
if (latencies.Count == 1000000)
{
PrintLatencyReport(latencies);
}
//Console.WriteLine("");
//Console.WriteLine(latencies.Count);
};
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topics.First(), 0, 5)
});
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
//Console.ReadLine();
//PrintLatencyReport(latencies);
Console.ReadLine();
}
}
public KafkaConsumer(string brokerlist, string topic)
{
var config = new Config {
GroupId = "group"
};
_consumer = new EventConsumer(config, brokerlist);
_topicName = topic;
_i = 0;
_consumer.OnMessage += (sender, message) =>
{
if (message.Payload.Length <= 0)
{
return;
}
_i++;
if (_i % 5000 == 0)
{
var msg = common.Message.Deserialize(message.Payload);
Console.WriteLine($"Topic {_topicName} received msg: {msg.Title} - {msg.Body} - {msg.DeviceId}");
}
};
//_consumer.Subscribe(new List<string>{topic});
_consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, Offset.End)
});
_consumer.Start();
}
public static void Main(string[] args)
{
string brokerList = "localhost";
var topics = args.Skip(1).ToList();
var config = new Config()
{
GroupId = "simple-csharp-consumer"
};
using (var consumer = new EventConsumer(config, brokerList))
{
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset("topicName", 0, 5)
});
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
public void Start()
{
var config = new Config()
{
GroupId = "simple-csharp-consumer"
};
using (var consumer = new EventConsumer(config, "")) {
consumer.OnMessage += (obj, msg) => {
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topics.First(), 0, 5)
});
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
Console.ReadLine();
}
Console.WriteLine("Starting...");
}
public static void Main(string[] args)
{
using (var producer = new Producer("localhost:9092"))
using (var topic = producer.Topic("test"))
{
var data = Encoding.UTF8.GetBytes("Hello Kafka");
var deliveryReport = topic.Produce(data).Result;
Console.WriteLine($"Produced to Partition: {deliveryReport.Partition}, Offset: {deliveryReport.Offset}");
}
var config = new Config {
GroupId = "test-consumer"
};
using (var consumer = new EventConsumer(config, "localhost:9092"))
{
consumer.OnMessage += (obj, msg) =>
{
var text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
consumer.Subscribe(new List <string> {
"test"
});
consumer.Start();
Console.ReadLine();
}
}
public static void Main(string[] args)
{
//string brokerList = "40.118.249.252";
string brokerList = "138.91.140.244";
//string brokerList = "localhost";
List <String> topicList = new List <string>();
topicList.Add("testLikeWin");
var topics = topicList;
var config = new Config()
{
GroupId = "simple-csharp-consumer"
};
using (var consumer = new EventConsumer(config, brokerList))
{
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topics.First(), 0, 5)
});
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
static void Main(string[] args)
{
Console.OutputEncoding = Encoding.UTF8;
Dictionary <string, List <YuanXinLogContext> > dictionary = new Dictionary <string, List <YuanXinLogContext> >();
//配置消费者组
try
{
var config = new Config()
{
GroupId = "example-csharp-consumer"
};
using (var consumer = new EventConsumer(config, ""))
{
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
var myclient = msg.Payload.DeSerialize <YuanXinLogContext>();
var key = myclient.ContextId;
if (dictionary.ContainsKey(key))
{
List <YuanXinLogContext> list;
dictionary.TryGetValue(key, out list);
list.Add(myclient);
dictionary.Remove(key);
dictionary.Add(key, list);
}
else
{
dictionary.Add(key, new List <YuanXinLogContext> {
myclient
});
}
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset}");
};
//订阅一个或者多个Topic
var tops = new List <string>();
tops.Add("LoggerTopic");
consumer.Subscribe(tops);
//启动
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex}");
}
}
/// <summary>
/// 启动消息监听
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="topicNames"></param>
/// <param name="groupId">消费者组</param>
/// <param name="host"></param>
/// <param name="consumerAction">处理该类消息的逻辑类</param>
public void SubscribeAt <T>(List <string> topicNames, string groupId, string host, IEventConsumer <T> consumerAction)
{
var config = new Config()
{
GroupId = groupId
};
var consumer = new EventConsumer(config, $"{host}:9092");
consumer.OnMessage += (obj, msg) => { consumerAction.Consumer(msg); };
consumer.OnConsumerError += (obj, error) => { Log.Write($"消费消息异常:{error.ToString()}", MessageType.Error); };
consumer.Subscribe(topicNames);
consumer.Start();
}
public void StartConsumer()
{
if (!ConsumerStarted)
{
lock (o)
{
if (!ConsumerStarted)
{
ConsumerStarted = true;
consumer.Start();
}
}
}
}
public KafkaConsumer(ILogger logger, List <string> topics, IConfigurationSection kafkaConfig, MessageHandler handler)
{
_logger = logger;
_handler = handler;
var config = KafkaConfigReader.Read(kafkaConfig);
config.EnableAutoCommit = false;
KafkaConfigReader.LogConfig(_logger, config);
config.Logger = OnLog;
_consumer = new EventConsumer(config);
_consumer.Subscribe(topics);
_consumer.OnMessage += OnMessage;
logger.LogInformation("Starting consumer");
_consumer.Start();
}
public async Task Test1()
{
//Larva.MessageProcess.LoggerManager.SetLoggerProvider(new Log4NetLoggerProvider());
var consumer1 = new EventConsumer();
consumer1.Initialize(new ConsumerSettings
{
AmqpUri = new Uri("amqp://*****:*****@localhost/test")
}, "MessageProcess_EventTopic", 4, false, 1, null, typeof(DomainEventTests).Assembly);
consumer1.Start();
var consumer2 = new EventConsumer();
consumer2.Initialize(new ConsumerSettings
{
AmqpUri = new Uri("amqp://*****:*****@localhost/test"),
GroupName = "Subscriber2"
}, "MessageProcess_EventTopic", 4, true, 1, null, typeof(DomainEventTests).Assembly);
consumer2.Start();
var eventBus = new EventBus();
eventBus.Initialize(new ProducerSettings
{
AmqpUri = new Uri("amqp://*****:*****@localhost/test")
}, "MessageProcess_EventTopic");
eventBus.Start();
for (var i = 1; i <= 5; i++)
{
for (var j = 1; j <= 5; j++)
{
await eventBus.PublishAsync(new DomainEvent1($"Test{i}"));
}
}
Thread.Sleep(1000);
eventBus.Shutdown();
Thread.Sleep(10000);
consumer1.Shutdown();
consumer2.Shutdown();
}
public async Task TestAsync()
{
// Producer 接受一个或多个 BrokerList
using (Producer producer = new Producer("132.232.4.73:9092"))
//发送到一个名为 testtopic 的Topic，如果没有就会创建一个
using (Topic topic = producer.Topic("testtopic"))
{
//将message转为一个 byte[]
byte[] data = Encoding.UTF8.GetBytes("Hello RdKafka");
DeliveryReport deliveryReport = await topic.Produce(data);
Console.WriteLine($"发送到分区：{deliveryReport.Partition}, Offset 为: {deliveryReport.Offset}");
}
//配置消费者组
var config = new Config()
{
GroupId = "example-csharp-consumer"
};
using (var consumer = new EventConsumer(config, "132.232.4.73:9092"))
{
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
//订阅一个或者多个Topic
consumer.Subscribe(new[] { "testtopic" });
//启动
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
// WIP, not producing useful numbers yet. Assumes one partition.
public static async Task <long> Consume(string broker, string topic)
{
long n = 0;
var topicConfig = new TopicConfig();
topicConfig["auto.offset.reset"] = "smallest";
var config = new Config()
{
GroupId = "benchmark-consumer",
DefaultTopicConfig = topicConfig
};
using (var consumer = new EventConsumer(config, broker))
{
var signal = new SemaphoreSlim(0, 1);
consumer.OnMessage += (obj, msg) =>
{
n += 1;
};
consumer.OnEndReached += (obj, end) =>
{
Console.WriteLine($"End reached");
signal.Release();
};
consumer.Subscribe(new List <string> {
topic
});
consumer.Start();
await signal.WaitAsync();
Console.WriteLine($"Shutting down");
}
return(n);
}
protected override void Retrieve(BrokerRouter mq, Func <string, bool> func, bool noAck)
{
var config = new RdKafka.Config()
{
GroupId = Consumer
};
using (var consumer = new EventConsumer(config, CreateMessageQueue().BrokerServer))
{
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
try
{
//msg.TopicPartitionOffset
func(msg.Payload.ToUtf8String());
}
catch (Exception ex)
{
Logger.Write(new LogExceptionEntity(ex));
if (ConfigEntity.ErrorThrowException)
{
throw new MessageQueueException(ex);
}
}
//Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
//订阅一个或者多个Topic
consumer.Subscribe(ConfigEntity.ExtentProperty[TopicNameKey].Split(',').ToList());
//启动
consumer.Start();
//Console.WriteLine("Started consumer, press enter to stop consuming");
//Console.ReadLine();
}
}
public static void BeginConsumer(string consumername, string topic)
{
//配置消费者组
var config = new Config()
{
GroupId = consumername, StatisticsInterval = new TimeSpan(1000)
};
var consumer = new EventConsumer(config, baseconfig.KafakServer);
//注册一个事件
consumer.OnMessage += (obj, msg) =>
{
try
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
};
//订阅一个或者多个Topic
consumer.Subscribe(new List <string>()
{
topic
});
//var message = consumer.Consume(new TimeSpan(1000));
//var msg = message.Value.Message;
//string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
//Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
//启动
consumer.Start();
Console.WriteLine("Started consumer, press enter to stop consuming");
//Console.ReadLine();
}
public static void Start(string kafkaUrl, string kafkaTopic)
{
Config config = new Config()
{
GroupId = "example-csharp-consumer"
};
config["security.protocol"] = "ssl";
config["ssl.ca.location"] = ConfigurationManager.AppSettings["ssl.ca.location"];
config["ssl.certificate.location"] = ConfigurationManager.AppSettings["ssl.certificate.location"];
config["ssl.key.location"] = ConfigurationManager.AppSettings["ssl.key.location"];
config["ssl.key.password"] = ConfigurationManager.AppSettings["ssl.key.password"];
consumer = new EventConsumer(config, kafkaUrl);
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
};
consumer.Subscribe(new List <string> {
kafkaTopic
});
consumer.Start();
}
public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory, EventConsumer eventConsumer)
{
eventConsumer.Start();
app.UseOwin(o => o.UseNancy());
}
public static void Run(string brokerList, List <string> topics)
{
bool enableAutoCommit = false;
var config = new Config()
{
GroupId = "advanced-csharp-consumer",
EnableAutoCommit = enableAutoCommit,
StatisticsInterval = TimeSpan.FromSeconds(60)
};
using (var consumer = new EventConsumer(config, brokerList))
{
consumer.OnMessage += (obj, msg) =>
{
string text = Encoding.UTF8.GetString(msg.Payload, 0, msg.Payload.Length);
Console.WriteLine($"Topic: {msg.Topic} Partition: {msg.Partition} Offset: {msg.Offset} {text}");
Console.WriteLine("1 Response: Partition {0},Offset {1} : {2}",
msg.Partition, msg.Offset, text);
if (!enableAutoCommit && msg.Offset % 10 == 0)
{
Console.WriteLine("Committing offset");
consumer.Commit(msg).Wait();
Console.WriteLine("Committed offset");
}
};
consumer.OnConsumerError += (obj, errorCode) =>
{
Console.WriteLine($"Consumer Error: {errorCode}");
};
consumer.OnEndReached += (obj, end) =>
{
Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
};
consumer.OnError += (obj, error) =>
{
Console.WriteLine($"Error: {error.ErrorCode} {error.Reason}");
};
if (enableAutoCommit)
{
consumer.OnOffsetCommit += (obj, commit) =>
{
if (commit.Error != ErrorCode.NO_ERROR)
{
Console.WriteLine($"Failed to commit offsets: {commit.Error}");
}
Console.WriteLine($"Successfully committed offsets: [{string.Join(", ", commit.Offsets)}]");
};
}
consumer.OnPartitionsAssigned += (obj, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}");
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (obj, partitions) =>
{
Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]");
consumer.Unassign();
};
consumer.OnStatistics += (obj, json) =>
{
Console.WriteLine($"Statistics: {json}");
};
consumer.Subscribe(topics);
consumer.Start();
Console.WriteLine($"Assigned to: [{string.Join(", ", consumer.Assignment)}]");
Console.WriteLine($"Subscribed to: [{string.Join(", ", consumer.Subscription)}]");
Console.WriteLine($"Started consumer, press enter to stop consuming");
Console.ReadLine();
}
}
public void SubscribeTopic <T, TH>(Func <TH> handler, string borkerList, string groupID, List <string> topics)
where TH : IProcessMessage <T>
where T : MessageBase
{
var containsKey = _subsManager.HasSubscriptionsForEvent <T>();
if (!containsKey)
{
Task.Run(() =>
{
try
{
if (string.IsNullOrWhiteSpace(borkerList))
{
borkerList = ConnectionBuilder.getBorkerList();
}
bool enableAutoCommit = false;
var config = new Config()
{
GroupId = groupID,
EnableAutoCommit = enableAutoCommit,
StatisticsInterval = TimeSpan.FromSeconds(60)
};
EventConsumer consumer = new EventConsumer(config, borkerList);
consumer.OnMessage += async(obj, msgs) => {
string text = Encoding.UTF8.GetString(msgs.Payload, 0, msgs.Payload.Length);
_logger.LogInformation("Topic: {0} Partition: {1} Offset: {2} {3}", msgs.Topic, msgs.Partition, msgs.Offset, text);
MessageBase mb = new MessageBase()
{
MessageBodyByte = msgs.Payload,
MessageRouter = msgs.Topic
};
await ProcessEvent(mb.MessageRouter, mb);
if (!enableAutoCommit && msgs.Offset % 10 == 0)
{
_logger.LogInformation($"Committing offset");
consumer.Commit(msgs).Wait();
_logger.LogInformation($"Committed offset");
}
};
consumer.OnConsumerError += (obj, errorCode) =>
{
_logger.LogWarning("Consumer Error: {0}", errorCode);
};
consumer.OnEndReached += (obj, end) => {
_logger.LogInformation("Reached end of topic {0} partition {1}, next message will be at offset {2}", end.Topic, end.Partition, end.Offset);
};
consumer.OnError += (obj, error) => {
_logger.LogWarning("Error: {0} {1}", error.ErrorCode, error.Reason);
};
if (enableAutoCommit)
{
consumer.OnOffsetCommit += (obj, commit) => {
if (commit.Error != ErrorCode.NO_ERROR)
{
_logger.LogInformation("Failed to commit offsets: {0}", commit.Error);
}
_logger.LogInformation("Successfully committed offsets: [{0}]", string.Join(", ", commit.Offsets));
};
}
consumer.OnPartitionsAssigned += (obj, partitions) => {
_logger.LogInformation("Assigned partitions: [{0}], member id: {1}", string.Join(", ", partitions), consumer.MemberId);
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (obj, partitions) => {
_logger.LogInformation("Revoked partitions: [{0}]", string.Join(", ", partitions));
consumer.Unassign();
};
consumer.OnStatistics += (obj, json) => {
_logger.LogInformation("Statistics: {0}", json);
};
consumer.Subscribe(topics);
consumer.Start();
_logger.LogInformation("Assigned to: [{0}]", string.Join(", ", consumer.Assignment));
_logger.LogInformation("Subscribed to: [{0}]", string.Join(", ", consumer.Subscription));
_logger.LogInformation("Started consumer, press enter to stop consuming");
while (true)
{
Thread.Sleep(1);
}
// Console.ReadLine();
}
catch (Exception ex)
{
_logger.LogError("SubscribeTopicAsync", ex);
return(ex.Message);
}
});//----------end run
}//-----------end if
_subsManager.AddSubscription <T, TH>(handler);
}//-----------end SubscribeTopic
