private static void RegularStatistics(ProduceSimpleHelperOption produceroundrobinOptions, int sentBatchCount) { ts = stopWatch.Elapsed; double mbTransferred = successMessageCount * (produceroundrobinOptions.MessageSize / 1048576.0); string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("Run: {0} seconds, {1:0.00} MB, {2:0.00} MB/s, {3:0.00} Success message/s \r\nRoudTrip for batch: {4:0.00}ms \r\nRoundtrip for message: {5:0.00}ms", ts.TotalSeconds, mbTransferred, mbTransferred / ts.TotalSeconds, ((double)successMessageCount) / ts.TotalSeconds, ts.TotalMilliseconds / sentBatchCount, ts.TotalMilliseconds / (successMessageCount)); Console.WriteLine("Success message count:{0} Failed message count: {1}", successMessageCount, failedMessageCount); if (produceroundrobinOptions.PartitionId == -2) { foreach (var k in produceMessagePerPartitionExpect.Keys.OrderBy(r => r)) { Console.WriteLine("Partition:{0} Expect Sent Message:{1} ", k, produceMessagePerPartitionExpect[k]); } } else { foreach (var k in produceMessagePerPartition.Keys.OrderBy(r => r)) { Console.WriteLine("Partition:{0} Sent Message:{1} ", k, produceMessagePerPartition[k]); } } }
internal static void Run(ProduceSimpleHelperOption produceOptions) { failedMessageCount = 0; successMessageCount = 0; sentBatchCount = 0; produceMessagePerPartition = new Dictionary <int, int>(); produceMessagePerPartitionExpect = new Dictionary <int, int>(); PrepareSentMessages(produceOptions); kafkaSimpleManagerConfig = new KafkaSimpleManagerConfiguration() { Zookeeper = produceOptions.Zookeeper, MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize, PartitionerClass = produceOptions.PartitionerClass }; kafkaSimpleManagerConfig.Verify(); producerConfigTemplate = new ProducerConfiguration( new List <BrokerConfiguration>() { }) //The Brokers will be replaced inside of KafkaSimpleManager { ForceToPartition = -1, PartitionerClass = kafkaSimpleManagerConfig.PartitionerClass, TotalNumPartitions = 0, RequiredAcks = produceOptions.RequiredAcks, AckTimeout = produceOptions.AckTimeout, SendTimeout = produceOptions.SendTimeout, ReceiveTimeout = produceOptions.ReceiveTimeout, CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()), BufferSize = produceOptions.BufferSize, SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize) }; using (ZooKeeperClient zkClient = new ZooKeeperClient(produceOptions.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); Dictionary <int, int[]> topicDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, produceOptions.Topic); // -2 by default or customized partitioner class. (you need specify PartitionerClass by -l) if (produceOptions.PartitionId == -2) { if (string.IsNullOrEmpty(kafkaSimpleManagerConfig.PartitionerClass)) { throw new ArgumentException("The partitioer class must not be empty if you want to send to partition by partitioner."); } //if (producerConfigTemplate.TotalNumPartitions <= 0) // throw new ArgumentException(string.Format("Please also specify the TotalNumPartitions if you want to send to partition by partitioner.")); ProduceByPartitionerClass(produceOptions, topicDataInZookeeper.Count); } else { ProduceToRandomOrSpecificPartition(produceOptions); } } }
private static void PrepareSentMessages(ProduceSimpleHelperOption produceroundrobinOptions) { Console.WriteLine("start perf test, {0} ", DateTime.Now); listOfDataNeedSendInOneBatch = new List <ProducerData <byte[], Message> >(); for (int i = 0; i < produceroundrobinOptions.MessageCountPerBatch; i++) { String val = KafkaClientHelperUtils.GetRandomString(produceroundrobinOptions.MessageSize); byte[] bVal = System.Text.Encoding.UTF8.GetBytes(val); byte[] bKey = System.Text.Encoding.UTF8.GetBytes(string.Format("{0:000000}", i)); if (produceroundrobinOptions.ConstantMessageKey) { bKey = System.Text.Encoding.UTF8.GetBytes(string.Format("{0:000000}", 0)); } Message message = new Message(bVal, bKey, produceroundrobinOptions.CompressionCodec); listOfKeys.Add(bKey); listOfDataNeedSendInOneBatch.Add(new ProducerData <byte[], Message>(produceroundrobinOptions.Topic, bKey, message)); } }
private static void ProduceToRandomOrSpecificPartition(ProduceSimpleHelperOption produceOptions) { using (KafkaSimpleManager <byte[], Message> kafkaSimpleManager = new KafkaSimpleManager <byte[], Message>(kafkaSimpleManagerConfig)) { Stopwatch stopwatch2 = new Stopwatch(); stopwatch2.Start(); int nProducers = kafkaSimpleManager.InitializeProducerPoolForTopic(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true, producerConfigTemplate, false); stopwatch2.Stop(); Console.WriteLine("Spent {0:0.00} ms to create {1} producers. average {2:0.00} ms per producer. syncProducerOfOnePartition:{3} ", stopwatch2.Elapsed.TotalMilliseconds, nProducers, stopwatch2.Elapsed.TotalMilliseconds / nProducers, producerConfigTemplate.SyncProducerOfOneBroker); totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count(); int targetPartitionID = Int16.MinValue; while (true) { Producer <byte[], Message> producer = null; try { targetPartitionID = produceOptions.PartitionId >= 0 ? produceOptions.PartitionId : rand.Next(totalPartitionCount); Logger.InfoFormat("Will try get producer for partition:{0} ", targetPartitionID); producer = kafkaSimpleManager.GetProducerOfPartition(produceOptions.Topic, targetPartitionID, false); Logger.InfoFormat("Get producer for partition:{0} Producer:{1} ", targetPartitionID, producer == null ? "null" : producer.ToString()); stopWatch.Start(); producer.Send(listOfDataNeedSendInOneBatch); successMessageCount += produceOptions.MessageCountPerBatch; if (!produceMessagePerPartition.ContainsKey(targetPartitionID)) { produceMessagePerPartition.Add(targetPartitionID, 0); } produceMessagePerPartition[targetPartitionID] = produceMessagePerPartition[targetPartitionID] + listOfDataNeedSendInOneBatch.Count; stopWatch.Stop(); sentBatchCount++; if (sentBatchCount % 10 == 0) { RegularStatistics(produceOptions, sentBatchCount); } if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount) { break; } //Maybe partition increased, need regular check if partition number has changed. if (DateTime.UtcNow > lastTimeRefreshMetadata.AddMinutes(10)) { List <string> partitoins = kafkaSimpleManager.GetTopicPartitionsFromZK(produceOptions.Topic); if (partitoins.Count != targetPartitionID) { nProducers = kafkaSimpleManager.InitializeProducerPoolForTopic(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true, producerConfigTemplate, false); totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count(); targetPartitionID = produceOptions.PartitionId >= 0 ? produceOptions.PartitionId : rand.Next(totalPartitionCount); Logger.InfoFormat("Will try get producer for partition:{0} ", targetPartitionID); producer = kafkaSimpleManager.GetProducerOfPartition(produceOptions.Topic, targetPartitionID, false); Logger.InfoFormat("Get producer for partition:{0} Producer:{1} ", targetPartitionID, producer == null ? "null" : producer.ToString()); } lastTimeRefreshMetadata = DateTime.UtcNow; } } catch (Exception e) { Logger.ErrorFormat("Got exception, maybe leader change or not available for some partition, will refresh metadata and recreate producer {0}", e.FormatException()); try { producer = kafkaSimpleManager.RefreshMetadataAndRecreateProducerOfOnePartition(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, targetPartitionID, true, true, producerConfigTemplate, false); } catch (Exception ex) { Logger.ErrorFormat("Got exception while RefreshMetadataAndRecreateProducerOfOnePartition, maybe leader change for some partition, will refresh metadata and recreate producer {0} after 3 seconds ...", ex.FormatException()); } totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count(); if (targetPartitionID >= totalPartitionCount) { targetPartitionID = produceOptions.PartitionId >= 0 ? produceOptions.PartitionId : rand.Next(totalPartitionCount); Logger.InfoFormat("Will try get producer for partition:{0} ", targetPartitionID); producer = kafkaSimpleManager.GetProducerOfPartition(produceOptions.Topic, targetPartitionID, false); Logger.InfoFormat("Get producer for partition:{0} Producer:{1} ", targetPartitionID, producer == null ? "null" : producer.ToString()); } } if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount) { break; } } } RegularStatistics(produceOptions, sentBatchCount); }
private static void ProduceByPartitionerClass(ProduceSimpleHelperOption produceOptions, int partitionCountInZookeeper) { using (KafkaSimpleManager <byte[], Message> kafkaSimpleManager = new KafkaSimpleManager <byte[], Message>(kafkaSimpleManagerConfig)) { Stopwatch stopwatch2 = new Stopwatch(); stopwatch2.Start(); int nProducers = kafkaSimpleManager.InitializeProducerPoolForTopic(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true, producerConfigTemplate, false); stopwatch2.Stop(); Console.WriteLine("Spent {0:0.00} ms to create {1} producers. average {2:0.00} ms per producer. syncProducerOfOnePartition:{3} ", stopwatch2.Elapsed.TotalMilliseconds, nProducers, stopwatch2.Elapsed.TotalMilliseconds / nProducers, producerConfigTemplate.SyncProducerOfOneBroker); while (true) { Producer <byte[], Message> producer = null; try { producer = kafkaSimpleManager.GetProducerWithPartionerClass(produceOptions.Topic); //Here the totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count(); Logger.InfoFormat("Get GetProducerWithPartionerClass Producer:{0} totalPartitionCount has metadata:{1} partitionCountInZookeeper:{2} " , producer == null ? "null" : producer.ToString(), totalPartitionCount, partitionCountInZookeeper); stopWatch.Start(); CountExpectCount(listOfKeys, produceOptions.PartitionerClass, partitionCountInZookeeper); producer.Send(listOfDataNeedSendInOneBatch); successMessageCount += produceOptions.MessageCountPerBatch; stopWatch.Stop(); sentBatchCount++; if (sentBatchCount % 10 == 0) { RegularStatistics(produceOptions, sentBatchCount); } if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount) { break; } } catch (FailedToSendMessageException <byte[]> e) { //Sometime the sent maybe partially success. //For example, 100 message, averagely to 5 partitions, if one partitoin has no leader. // here will get the failed 20 message, you can retry or do something for them. Logger.Error("===FAILED========="); Logger.ErrorFormat("{0}", e.Message); if (e.ProduceDispatchSeralizeResult != null && e.ProduceDispatchSeralizeResult.FailedProducerDatas != null) { Logger.ErrorFormat("Failed produce message key: "); foreach (ProducerData <byte[], Message> a in e.ProduceDispatchSeralizeResult.FailedProducerDatas) { Logger.ErrorFormat("Key:{0} ", System.Text.Encoding.Default.GetString(a.Key)); } } //Here partially success we also consider failedMessageCount += e.CountFailed; successMessageCount += e.CountAll - e.CountFailed; sentBatchCount++; if (sentBatchCount % 10 == 0) { RegularStatistics(produceOptions, sentBatchCount); } if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount) { break; } Logger.Error(" \r\n"); } catch (Exception e) { Logger.ErrorFormat("Got exception, maybe leader change for some partition, will refresh metadata and recreate producer {0}", e.FormatException()); TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true); totalPartitionCount = topicMetadata.PartitionsMetadata.Count(); Logger.InfoFormat("Get GetProducerWithPartionerClass Producer:{0} totalPartitionCount has metadata:{1} partitionCountInZookeeper:{2} " , producer == null ? "null" : producer.ToString(), totalPartitionCount, partitionCountInZookeeper); } if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount) { break; } } } RegularStatistics(produceOptions, sentBatchCount); }
public static void MainInternal(int taskIndexInStorm, string[] args) { GenerateAssistFile("producesimple"); GenerateAssistFile("produceperftest"); GenerateAssistFile("producemonitor"); GenerateAssistFile("eventserverperftest"); GenerateAssistFile("consumesimple"); GenerateAssistFile("consumegroup"); GenerateAssistFile("consumegroupmonitor"); GenerateAssistFile("topic"); ServicePointManager.DefaultConnectionLimit = 5000; ServicePointManager.UseNagleAlgorithm = false; var log4netSection = ConfigurationManager.GetSection("log4net"); if (log4netSection != null) { //XmlConfigurator.Configure(); } KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb(); try { commandOptions.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage()); Environment.Exit(-1); } KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption; try { realOption.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(realOption.GetUsage(false)); Environment.Exit(-1); } Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict()); switch (KafkaNETExampleCommandVerb.Verb) { case "producesimple": case "produceroundrobin": ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption; ProduceSimpleHelper.Run(produceroundrobinOptions); break; case "produceperftest": case "producewrapper": ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption; (new ProducePerfTestHelper()).Run(producewrapperOption); break; case "producem": case "producemonitor": ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption; ProduceMonitorHelper.Run(produceMonitorOptions); break; case "eventserverperftest": JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption; (new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions); break; case "consumesimple": case "dumpdata": ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption; ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions); break; case "consumegroup": case "dumpdataasconsumergroup": ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption; if (taskIndexInStorm >= 0) { cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString(); cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File; } ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions); break; case "latestoffsetofconsumergroup": case "consumegroupmonitor": case "consumegroupm": case "consumem": ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption; ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions); break; case "topic": TopicHelperArguments dtOptions = (TopicHelperArguments)realOption; TopicHelper.DumpTopicMetadataAndOffset(dtOptions); break; case "test": var testOptions = (TestHelperOptions)realOption; TestHelper.Run(testOptions); break; default: Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb)); return; } }
internal void Parse(string[] args) { if (args == null || args.Length <= 0) { throw new ArgumentException("Please provide verb."); } Verb = args[0].ToLowerInvariant(); if ("topic" == Verb.ToLowerInvariant()) ActiveSubOption = new TopicHelperArguments(); else if ("dumpdata" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeDataHelperArguments(); else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupHelperOptions(); else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant() || "consumegroupm" == Verb.ToLowerInvariant() || "consumem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupMonitorHelperOptions(); else if ("produceroundrobin" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceSimpleHelperOption(); else if ("test" == Verb.ToLowerInvariant()) ActiveSubOption = new TestHelperOptions(); else if ("producewrapper" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProducePerfTestHelperOption(); else if ("producem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceMonitorHelperOptions(); else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new JavaEventServerPerfTestHelperOptions(); else { throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb)); } }