/// <summary> /// Initializes a new instance of the <see cref="PartitionTopicInfo"/> class. /// </summary> /// <param name="topic"> /// The topic. /// </param> /// <param name="partitionId"> /// The broker's partition. /// </param> /// <param name="chunkQueue"> /// The chunk queue. /// </param> /// <param name="consumedOffset"> /// The consumed offset value. /// </param> /// <param name="fetchedOffset"> /// The fetched offset value. /// </param> /// <param name="fetchSize"> /// The fetch size. /// </param> /// <param name="clientId">Client id</param> public PartitionTopicInfo( string topic, int partitionId, BlockingCollection<FetchedDataChunk> chunkQueue, AtomicLong consumedOffset, AtomicLong fetchedOffset, AtomicInteger fetchSize, string clientId) { this.Topic = topic; this.PartitionId = partitionId; this.chunkQueue = chunkQueue; this.consumedOffset = consumedOffset; this.fetchedOffset = fetchedOffset; this.fetchSize = fetchSize; this.clientId = clientId; this.consumerTopicStats = ConsumerTopicStatsRegistry.GetConsumerTopicStat(clientId); if (Logger.IsDebugEnabled) { Logger.DebugFormat( CultureInfo.CurrentCulture, "initial consumer offset of {0} is {1}", this, consumedOffset); Logger.DebugFormat( CultureInfo.CurrentCulture, "initial fetch offset of {0} is {1}", this, fetchedOffset); } }
public static void Main(string[] args) { var config = new ProducerPerfConfig(args); if (!config.IsFixedSize) { Logger.Warn("Throughput will be slower due to changing message size per request"); } var totalBytesSent = new AtomicLong(0); var totalMessagesSent = new AtomicLong(0); var allDone = new CountdownEvent(config.NumThreads); var start = DateTime.Now; var rand = new Random(); if (!config.HideHeader) { Console.WriteLine("start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, " + "total.data.sent.in.nMsg, nMsg.sec"); } for (var i = 0; i < config.NumThreads; i++) { new Thread(() => new ProducerThread(i, config, totalBytesSent, totalMessagesSent, allDone, rand).Run()).Start(); } allDone.Wait(); var end = DateTime.Now; var elapsedSecs = (end - start).TotalSeconds; var totalMBSent = (totalBytesSent.Get() * 1.0) / (1024 * 1024); Console.WriteLine( "{0}; {1}; {2}; {3}; {4}; {5:f2}; {6:f4}; {7}; {8:f4}", start.ToString(config.DateFormat), end.ToString(config.DateFormat), config.CompressionCodec, config.MessageSize, config.BatchSize, totalMBSent, totalMBSent / elapsedSecs, totalMessagesSent.Get(), totalMessagesSent.Get() / elapsedSecs); }
public ConsumerPerfThread(int threadId, string name, KafkaStream<byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead) { this.threadId = threadId; this.name = name; this.stream = stream; this.config = config; this.totalMessagesRead = totalMessagesRead; this.totalBytesRead = totalBytesRead; }
public static void Main(string[] args) { var config = new ConsumerPerfConfig(args); Logger.Info("Starting consumer..."); var totalMessagesRead = new AtomicLong(0); var totalBytesRead = new AtomicLong(0); if (!config.HideHeader) { if (!config.ShowDetailedStats) { Console.WriteLine( "start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec"); } else { Console.WriteLine("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec"); } } // clean up zookeeper state for this group id for every perf run ZkUtils.MaybeDeletePath(config.ConsumerConfig.ZooKeeper.ZkConnect, "/consumers/" + config.ConsumerConfig.GroupId); var consumerConnector = Client.Consumers.Consumer.Create(config.ConsumerConfig); var topicMessageStreams = consumerConnector.CreateMessageStreams( new Dictionary<string, int> { { config.Topic, config.NumThreads } }); var threadList = new List<ConsumerPerfThread>(); foreach (var topicAndMessageStream in topicMessageStreams) { var streamList = topicAndMessageStream.Value; for (var i = 0; i < streamList.Count; i++) { threadList.Add(new ConsumerPerfThread(i, "kafka-zk-consumer-" + i, streamList[i], config, totalMessagesRead, totalBytesRead)); } Logger.Info("Sleeping for 1 second."); Thread.Sleep(1000); Logger.Info("Starting threads."); var startMs = DateTime.Now; foreach (var thread in threadList) { new Thread(() => thread.Run()).Start(); } foreach (var thread in threadList) { thread.Shutdown(); } var endMs = DateTime.Now; var elapsedSecs = (endMs - startMs - TimeSpan.FromMilliseconds(config.ConsumerConfig.ConsumerTimeoutMs)).TotalMilliseconds / 1000.0; if (!config.ShowDetailedStats) { var totalMBRead = (totalBytesRead.Get() * 1.0) / (1024 * 1024); Console.WriteLine( "{0}, {1}, {2}, {3:f4}, {4:f4}, {5}, {6:f4}", startMs.ToString(config.DateFormat), endMs.ToString(config.DateFormat), config.ConsumerConfig.FetchMessageMaxBytes, totalMBRead, totalMBRead / elapsedSecs, totalMessagesRead.Get(), totalMessagesRead.Get() / elapsedSecs); } } Environment.Exit(0); }
public ProducerThread(int threadId, ProducerPerfConfig config, AtomicLong totalBytesSent, AtomicLong totalMessagesSent, CountdownEvent allDone, Random rand) { this.threadId = threadId; this.config = config; this.totalBytesSent = totalBytesSent; this.totalMessagesSent = totalMessagesSent; this.allDone = allDone; this.rand = rand; this.producerConfig = new ProducerConfig(); this.producerConfig.Brokers = config.BrokerList.Split(',') .Select( (s, idx) => new BrokerConfiguration { BrokerId = idx, Host = s.Split(':')[0], Port = int.Parse(s.Split(':')[1]) }).ToList(); this.producerConfig.CompressionCodec = config.CompressionCodec; this.producerConfig.SendBufferBytes = 64 * 1024; if (!config.IsSync) { this.producerConfig.ProducerType = ProducerTypes.Async; this.producerConfig.BatchNumMessages = config.BatchSize; this.producerConfig.QueueEnqueueTimeoutMs = -1; } this.producerConfig.ClientId = "ProducerPerformance"; this.producerConfig.RequestRequiredAcks = config.ProducerRequestRequiredAcks; this.producerConfig.RequestTimeoutMs = config.ProducerRequestTimeoutMs; this.producerConfig.MessageSendMaxRetries = config.ProducerNumRetries; this.producerConfig.RetryBackoffMs = config.ProducerRetryBackoffMs; this.producerConfig.Serializer = typeof(DefaultEncoder).AssemblyQualifiedName; this.producerConfig.KeySerializer = typeof(NullEncoder<long>).AssemblyQualifiedName; this.producer = new Producer<long, byte[]>(this.producerConfig); this.messagesPerThread = config.NumMessages / config.NumThreads; Logger.DebugFormat("Messages per thread = {0}", this.messagesPerThread); }