public ConsumerPerfThread(int threadId, string name, KafkaStream <byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead)
{
this.threadId = threadId;
this.name = name;
this.stream = stream;
this.config = config;
this.totalMessagesRead = totalMessagesRead;
this.totalBytesRead = totalBytesRead;
}
public ConsumerPerfThread(int threadId, string name, KafkaStream<byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead)
{
this.threadId = threadId;
this.name = name;
this.stream = stream;
this.config = config;
this.totalMessagesRead = totalMessagesRead;
this.totalBytesRead = totalBytesRead;
}
public static void Main(string[] args)
{
var config = new ConsumerPerfConfig(args);
Logger.Info("Starting consumer...");
var totalMessagesRead = new AtomicLong(0);
var totalBytesRead = new AtomicLong(0);
if (!config.HideHeader)
{
if (!config.ShowDetailedStats)
{
Console.WriteLine(
"start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
else
{
Console.WriteLine("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
}
// clean up zookeeper state for this group id for every perf run
ZkUtils.MaybeDeletePath(config.ConsumerConfig.ZooKeeper.ZkConnect, "/consumers/" + config.ConsumerConfig.GroupId);
var consumerConnector = Client.Consumers.Consumer.Create(config.ConsumerConfig);
var topicMessageStreams =
consumerConnector.CreateMessageStreams(
new Dictionary<string, int> { { config.Topic, config.NumThreads } });
var threadList = new List<ConsumerPerfThread>();
foreach (var topicAndMessageStream in topicMessageStreams)
{
var streamList = topicAndMessageStream.Value;
for (var i = 0; i < streamList.Count; i++)
{
threadList.Add(new ConsumerPerfThread(i, "kafka-zk-consumer-" + i, streamList[i], config, totalMessagesRead, totalBytesRead));
}
Logger.Info("Sleeping for 1 second.");
Thread.Sleep(1000);
Logger.Info("Starting threads.");
var startMs = DateTime.Now;
foreach (var thread in threadList)
{
new Thread(() => thread.Run()).Start();
}
foreach (var thread in threadList)
{
thread.Shutdown();
}
var endMs = DateTime.Now;
var elapsedSecs = (endMs - startMs - TimeSpan.FromMilliseconds(config.ConsumerConfig.ConsumerTimeoutMs)).TotalMilliseconds / 1000.0;
if (!config.ShowDetailedStats)
{
var totalMBRead = (totalBytesRead.Get() * 1.0) / (1024 * 1024);
Console.WriteLine(
"{0}, {1}, {2}, {3:f4}, {4:f4}, {5}, {6:f4}",
startMs.ToString(config.DateFormat),
endMs.ToString(config.DateFormat),
config.ConsumerConfig.FetchMessageMaxBytes,
totalMBRead,
totalMBRead / elapsedSecs,
totalMessagesRead.Get(),
totalMessagesRead.Get() / elapsedSecs);
}
}
Environment.Exit(0);
}
public static void Main(string[] args)
{
var config = new ConsumerPerfConfig(args);
Logger.Info("Starting consumer...");
var totalMessagesRead = new AtomicLong(0);
var totalBytesRead = new AtomicLong(0);
if (!config.HideHeader)
{
if (!config.ShowDetailedStats)
{
Console.WriteLine(
"start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
else
{
Console.WriteLine("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
}
// clean up zookeeper state for this group id for every perf run
ZkUtils.MaybeDeletePath(config.ConsumerConfig.ZooKeeper.ZkConnect, "/consumers/" + config.ConsumerConfig.GroupId);
var consumerConnector = Consumer.Create(config.ConsumerConfig);
var topicMessageStreams =
consumerConnector.CreateMessageStreams(
new Dictionary <string, int> {
{ config.Topic, config.NumThreads }
});
var threadList = new List <ConsumerPerfThread>();
foreach (var topicAndMessageStream in topicMessageStreams)
{
var streamList = topicAndMessageStream.Value;
for (var i = 0; i < streamList.Count; i++)
{
threadList.Add(new ConsumerPerfThread(i, "kafka-zk-consumer-" + i, streamList[i], config, totalMessagesRead, totalBytesRead));
}
Logger.Info("Sleeping for 1 second.");
Thread.Sleep(1000);
Logger.Info("Starting threads.");
var startMs = DateTime.Now;
foreach (var thread in threadList)
{
new Thread(() => thread.Run()).Start();
}
foreach (var thread in threadList)
{
thread.Shutdown();
}
var endMs = DateTime.Now;
var elapsedSecs = (endMs - startMs - TimeSpan.FromMilliseconds(config.ConsumerConfig.ConsumerTimeoutMs)).TotalMilliseconds / 1000.0;
if (!config.ShowDetailedStats)
{
var totalMBRead = (totalBytesRead.Get() * 1.0) / (1024 * 1024);
Console.WriteLine(
"{0}, {1}, {2}, {3:f4}, {4:f4}, {5}, {6:f4}",
startMs.ToString(config.DateFormat),
endMs.ToString(config.DateFormat),
config.ConsumerConfig.FetchMessageMaxBytes,
totalMBRead,
totalMBRead / elapsedSecs,
totalMessagesRead.Get(),
totalMessagesRead.Get() / elapsedSecs);
}
}
Environment.Exit(0);
}