/// <summary>
/// Initializes a new instance of the <see cref="PartitionTopicInfo"/> class.
/// </summary>
/// <param name="topic">
/// The topic.
/// </param>
/// <param name="partitionId">
/// The broker's partition.
/// </param>
/// <param name="chunkQueue">
/// The chunk queue.
/// </param>
/// <param name="consumedOffset">
/// The consumed offset value.
/// </param>
/// <param name="fetchedOffset">
/// The fetched offset value.
/// </param>
/// <param name="fetchSize">
/// The fetch size.
/// </param>
/// <param name="clientId">Client id</param>
public PartitionTopicInfo(
string topic,
int partitionId,
BlockingCollection<FetchedDataChunk> chunkQueue,
AtomicLong consumedOffset,
AtomicLong fetchedOffset,
AtomicInteger fetchSize,
string clientId)
{
this.Topic = topic;
this.PartitionId = partitionId;
this.chunkQueue = chunkQueue;
this.consumedOffset = consumedOffset;
this.fetchedOffset = fetchedOffset;
this.fetchSize = fetchSize;
this.clientId = clientId;
this.consumerTopicStats = ConsumerTopicStatsRegistry.GetConsumerTopicStat(clientId);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
CultureInfo.CurrentCulture, "initial consumer offset of {0} is {1}", this, consumedOffset);
Logger.DebugFormat(
CultureInfo.CurrentCulture, "initial fetch offset of {0} is {1}", this, fetchedOffset);
}
}
public static void Main(string[] args)
{
var config = new ProducerPerfConfig(args);
if (!config.IsFixedSize)
{
Logger.Warn("Throughput will be slower due to changing message size per request");
}
var totalBytesSent = new AtomicLong(0);
var totalMessagesSent = new AtomicLong(0);
var allDone = new CountdownEvent(config.NumThreads);
var start = DateTime.Now;
var rand = new Random();
if (!config.HideHeader)
{
Console.WriteLine("start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, " +
"total.data.sent.in.nMsg, nMsg.sec");
}
for (var i = 0; i < config.NumThreads; i++)
{
new Thread(() => new ProducerThread(i, config, totalBytesSent, totalMessagesSent, allDone, rand).Run()).Start();
}
allDone.Wait();
var end = DateTime.Now;
var elapsedSecs = (end - start).TotalSeconds;
var totalMBSent = (totalBytesSent.Get() * 1.0) / (1024 * 1024);
Console.WriteLine(
"{0}; {1}; {2}; {3}; {4}; {5:f2}; {6:f4}; {7}; {8:f4}",
start.ToString(config.DateFormat),
end.ToString(config.DateFormat),
config.CompressionCodec,
config.MessageSize,
config.BatchSize,
totalMBSent,
totalMBSent / elapsedSecs,
totalMessagesSent.Get(),
totalMessagesSent.Get() / elapsedSecs);
}
public ConsumerPerfThread(int threadId, string name, KafkaStream<byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead)
{
this.threadId = threadId;
this.name = name;
this.stream = stream;
this.config = config;
this.totalMessagesRead = totalMessagesRead;
this.totalBytesRead = totalBytesRead;
}
public static void Main(string[] args)
{
var config = new ConsumerPerfConfig(args);
Logger.Info("Starting consumer...");
var totalMessagesRead = new AtomicLong(0);
var totalBytesRead = new AtomicLong(0);
if (!config.HideHeader)
{
if (!config.ShowDetailedStats)
{
Console.WriteLine(
"start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
else
{
Console.WriteLine("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec");
}
}
// clean up zookeeper state for this group id for every perf run
ZkUtils.MaybeDeletePath(config.ConsumerConfig.ZooKeeper.ZkConnect, "/consumers/" + config.ConsumerConfig.GroupId);
var consumerConnector = Client.Consumers.Consumer.Create(config.ConsumerConfig);
var topicMessageStreams =
consumerConnector.CreateMessageStreams(
new Dictionary<string, int> { { config.Topic, config.NumThreads } });
var threadList = new List<ConsumerPerfThread>();
foreach (var topicAndMessageStream in topicMessageStreams)
{
var streamList = topicAndMessageStream.Value;
for (var i = 0; i < streamList.Count; i++)
{
threadList.Add(new ConsumerPerfThread(i, "kafka-zk-consumer-" + i, streamList[i], config, totalMessagesRead, totalBytesRead));
}
Logger.Info("Sleeping for 1 second.");
Thread.Sleep(1000);
Logger.Info("Starting threads.");
var startMs = DateTime.Now;
foreach (var thread in threadList)
{
new Thread(() => thread.Run()).Start();
}
foreach (var thread in threadList)
{
thread.Shutdown();
}
var endMs = DateTime.Now;
var elapsedSecs = (endMs - startMs - TimeSpan.FromMilliseconds(config.ConsumerConfig.ConsumerTimeoutMs)).TotalMilliseconds / 1000.0;
if (!config.ShowDetailedStats)
{
var totalMBRead = (totalBytesRead.Get() * 1.0) / (1024 * 1024);
Console.WriteLine(
"{0}, {1}, {2}, {3:f4}, {4:f4}, {5}, {6:f4}",
startMs.ToString(config.DateFormat),
endMs.ToString(config.DateFormat),
config.ConsumerConfig.FetchMessageMaxBytes,
totalMBRead,
totalMBRead / elapsedSecs,
totalMessagesRead.Get(),
totalMessagesRead.Get() / elapsedSecs);
}
}
Environment.Exit(0);
}
public ProducerThread(int threadId, ProducerPerfConfig config, AtomicLong totalBytesSent, AtomicLong totalMessagesSent, CountdownEvent allDone, Random rand)
{
this.threadId = threadId;
this.config = config;
this.totalBytesSent = totalBytesSent;
this.totalMessagesSent = totalMessagesSent;
this.allDone = allDone;
this.rand = rand;
this.producerConfig = new ProducerConfig();
this.producerConfig.Brokers =
config.BrokerList.Split(',')
.Select(
(s, idx) =>
new BrokerConfiguration
{
BrokerId = idx,
Host = s.Split(':')[0],
Port = int.Parse(s.Split(':')[1])
}).ToList();
this.producerConfig.CompressionCodec = config.CompressionCodec;
this.producerConfig.SendBufferBytes = 64 * 1024;
if (!config.IsSync)
{
this.producerConfig.ProducerType = ProducerTypes.Async;
this.producerConfig.BatchNumMessages = config.BatchSize;
this.producerConfig.QueueEnqueueTimeoutMs = -1;
}
this.producerConfig.ClientId = "ProducerPerformance";
this.producerConfig.RequestRequiredAcks = config.ProducerRequestRequiredAcks;
this.producerConfig.RequestTimeoutMs = config.ProducerRequestTimeoutMs;
this.producerConfig.MessageSendMaxRetries = config.ProducerNumRetries;
this.producerConfig.RetryBackoffMs = config.ProducerRetryBackoffMs;
this.producerConfig.Serializer = typeof(DefaultEncoder).AssemblyQualifiedName;
this.producerConfig.KeySerializer = typeof(NullEncoder<long>).AssemblyQualifiedName;
this.producer = new Producer<long, byte[]>(this.producerConfig);
this.messagesPerThread = config.NumMessages / config.NumThreads;
Logger.DebugFormat("Messages per thread = {0}", this.messagesPerThread);
}
