public static void Main(string[] args) { var config = new ProducerPerfConfig(args); if (!config.IsFixedSize) { Logger.Warn("Throughput will be slower due to changing message size per request"); } var totalBytesSent = new AtomicLong(0); var totalMessagesSent = new AtomicLong(0); var allDone = new CountdownEvent(config.NumThreads); var start = DateTime.Now; var rand = new Random(); if (!config.HideHeader) { Console.WriteLine("start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, " + "total.data.sent.in.nMsg, nMsg.sec"); } for (var i = 0; i < config.NumThreads; i++) { new Thread(() => new ProducerThread(i, config, totalBytesSent, totalMessagesSent, allDone, rand).Run()).Start(); } allDone.Wait(); var end = DateTime.Now; var elapsedSecs = (end - start).TotalSeconds; var totalMBSent = (totalBytesSent.Get() * 1.0) / (1024 * 1024); Console.WriteLine( "{0}; {1}; {2}; {3}; {4}; {5:f2}; {6:f4}; {7}; {8:f4}", start.ToString(config.DateFormat), end.ToString(config.DateFormat), config.CompressionCodec, config.MessageSize, config.BatchSize, totalMBSent, totalMBSent / elapsedSecs, totalMessagesSent.Get(), totalMessagesSent.Get() / elapsedSecs); }
public static void Main(string[] args) { var config = new ConsumerPerfConfig(args); Logger.Info("Starting consumer..."); var totalMessagesRead = new AtomicLong(0); var totalBytesRead = new AtomicLong(0); if (!config.HideHeader) { if (!config.ShowDetailedStats) { Console.WriteLine( "start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec"); } else { Console.WriteLine("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec"); } } // clean up zookeeper state for this group id for every perf run ZkUtils.MaybeDeletePath(config.ConsumerConfig.ZooKeeper.ZkConnect, "/consumers/" + config.ConsumerConfig.GroupId); var consumerConnector = Client.Consumers.Consumer.Create(config.ConsumerConfig); var topicMessageStreams = consumerConnector.CreateMessageStreams( new Dictionary<string, int> { { config.Topic, config.NumThreads } }); var threadList = new List<ConsumerPerfThread>(); foreach (var topicAndMessageStream in topicMessageStreams) { var streamList = topicAndMessageStream.Value; for (var i = 0; i < streamList.Count; i++) { threadList.Add(new ConsumerPerfThread(i, "kafka-zk-consumer-" + i, streamList[i], config, totalMessagesRead, totalBytesRead)); } Logger.Info("Sleeping for 1 second."); Thread.Sleep(1000); Logger.Info("Starting threads."); var startMs = DateTime.Now; foreach (var thread in threadList) { new Thread(() => thread.Run()).Start(); } foreach (var thread in threadList) { thread.Shutdown(); } var endMs = DateTime.Now; var elapsedSecs = (endMs - startMs - TimeSpan.FromMilliseconds(config.ConsumerConfig.ConsumerTimeoutMs)).TotalMilliseconds / 1000.0; if (!config.ShowDetailedStats) { var totalMBRead = (totalBytesRead.Get() * 1.0) / (1024 * 1024); Console.WriteLine( "{0}, {1}, {2}, {3:f4}, {4:f4}, {5}, {6:f4}", startMs.ToString(config.DateFormat), endMs.ToString(config.DateFormat), config.ConsumerConfig.FetchMessageMaxBytes, totalMBRead, totalMBRead / elapsedSecs, totalMessagesRead.Get(), totalMessagesRead.Get() / elapsedSecs); } } Environment.Exit(0); }