public static void Main(string[] args)
{
var config = new ProducerPerfConfig(args);
if (!config.IsFixedSize)
{
Logger.Warn("Throughput will be slower due to changing message size per request");
}
var totalBytesSent = new AtomicLong(0);
var totalMessagesSent = new AtomicLong(0);
var allDone = new CountdownEvent(config.NumThreads);
var start = DateTime.Now;
var rand = new Random();
if (!config.HideHeader)
{
Console.WriteLine("start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, " +
"total.data.sent.in.nMsg, nMsg.sec");
}
for (var i = 0; i < config.NumThreads; i++)
{
new Thread(() => new ProducerThread(i, config, totalBytesSent, totalMessagesSent, allDone, rand).Run()).Start();
}
allDone.Wait();
var end = DateTime.Now;
var elapsedSecs = (end - start).TotalSeconds;
var totalMBSent = (totalBytesSent.Get() * 1.0) / (1024 * 1024);
Console.WriteLine(
"{0}; {1}; {2}; {3}; {4}; {5:f2}; {6:f4}; {7}; {8:f4}",
start.ToString(config.DateFormat),
end.ToString(config.DateFormat),
config.CompressionCodec,
config.MessageSize,
config.BatchSize,
totalMBSent,
totalMBSent / elapsedSecs,
totalMessagesSent.Get(),
totalMessagesSent.Get() / elapsedSecs);
}
public static void Main(string[] args)
{
var config = new ConsumerPerfConfig(args);
Logger.Info("Starting consumer...");
var totalMessagesRead = new AtomicLong(0);
var totalBytesRead = new AtomicLong(0);
if (!config.HideHeader)
{
if (!config.ShowDetailedStats)
{
Console.WriteLine(
"start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
else
{
Console.WriteLine("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
}
// clean up zookeeper state for this group id for every perf run
ZkUtils.MaybeDeletePath(config.ConsumerConfig.ZooKeeper.ZkConnect, "/consumers/" + config.ConsumerConfig.GroupId);
var consumerConnector = Client.Consumers.Consumer.Create(config.ConsumerConfig);
var topicMessageStreams =
consumerConnector.CreateMessageStreams(
new Dictionary<string, int> { { config.Topic, config.NumThreads } });
var threadList = new List<ConsumerPerfThread>();
foreach (var topicAndMessageStream in topicMessageStreams)
{
var streamList = topicAndMessageStream.Value;
for (var i = 0; i < streamList.Count; i++)
{
threadList.Add(new ConsumerPerfThread(i, "kafka-zk-consumer-" + i, streamList[i], config, totalMessagesRead, totalBytesRead));
}
Logger.Info("Sleeping for 1 second.");
Thread.Sleep(1000);
Logger.Info("Starting threads.");
var startMs = DateTime.Now;
foreach (var thread in threadList)
{
new Thread(() => thread.Run()).Start();
}
foreach (var thread in threadList)
{
thread.Shutdown();
}
var endMs = DateTime.Now;
var elapsedSecs = (endMs - startMs - TimeSpan.FromMilliseconds(config.ConsumerConfig.ConsumerTimeoutMs)).TotalMilliseconds / 1000.0;
if (!config.ShowDetailedStats)
{
var totalMBRead = (totalBytesRead.Get() * 1.0) / (1024 * 1024);
Console.WriteLine(
"{0}, {1}, {2}, {3:f4}, {4:f4}, {5}, {6:f4}",
startMs.ToString(config.DateFormat),
endMs.ToString(config.DateFormat),
config.ConsumerConfig.FetchMessageMaxBytes,
totalMBRead,
totalMBRead / elapsedSecs,
totalMessagesRead.Get(),
totalMessagesRead.Get() / elapsedSecs);
}
}
Environment.Exit(0);
}