private static void RegularStatistics(ProduceSimpleHelperOption produceroundrobinOptions, int sentBatchCount)
{
ts = stopWatch.Elapsed;
double mbTransferred = successMessageCount * (produceroundrobinOptions.MessageSize / 1048576.0);
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Console.WriteLine("Run: {0} seconds, {1:0.00} MB, {2:0.00} MB/s, {3:0.00} Success message/s \r\nRoudTrip for batch: {4:0.00}ms \r\nRoundtrip for message: {5:0.00}ms",
ts.TotalSeconds,
mbTransferred,
mbTransferred / ts.TotalSeconds,
((double)successMessageCount) / ts.TotalSeconds,
ts.TotalMilliseconds / sentBatchCount,
ts.TotalMilliseconds / (successMessageCount));
Console.WriteLine("Success message count:{0} Failed message count: {1}", successMessageCount, failedMessageCount);
if (produceroundrobinOptions.PartitionId == -2)
{
foreach (var k in produceMessagePerPartitionExpect.Keys.OrderBy(r => r))
{
Console.WriteLine("Partition:{0} Expect Sent Message:{1} ", k, produceMessagePerPartitionExpect[k]);
}
}
else
{
foreach (var k in produceMessagePerPartition.Keys.OrderBy(r => r))
{
Console.WriteLine("Partition:{0} Sent Message:{1} ", k, produceMessagePerPartition[k]);
}
}
}
internal static void Run(ProduceSimpleHelperOption produceOptions)
{
failedMessageCount = 0;
successMessageCount = 0;
sentBatchCount = 0;
produceMessagePerPartition = new Dictionary <int, int>();
produceMessagePerPartitionExpect = new Dictionary <int, int>();
PrepareSentMessages(produceOptions);
kafkaSimpleManagerConfig = new KafkaSimpleManagerConfiguration()
{
Zookeeper = produceOptions.Zookeeper,
MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize,
PartitionerClass = produceOptions.PartitionerClass
};
kafkaSimpleManagerConfig.Verify();
producerConfigTemplate = new ProducerConfiguration(
new List <BrokerConfiguration>()
{
}) //The Brokers will be replaced inside of KafkaSimpleManager
{
ForceToPartition = -1,
PartitionerClass = kafkaSimpleManagerConfig.PartitionerClass,
TotalNumPartitions = 0,
RequiredAcks = produceOptions.RequiredAcks,
AckTimeout = produceOptions.AckTimeout,
SendTimeout = produceOptions.SendTimeout,
ReceiveTimeout = produceOptions.ReceiveTimeout,
CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()),
BufferSize = produceOptions.BufferSize,
SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition
MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize)
};
using (ZooKeeperClient zkClient = new ZooKeeperClient(produceOptions.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
Dictionary <int, int[]> topicDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, produceOptions.Topic);
// -2 by default or customized partitioner class. (you need specify PartitionerClass by -l)
if (produceOptions.PartitionId == -2)
{
if (string.IsNullOrEmpty(kafkaSimpleManagerConfig.PartitionerClass))
{
throw new ArgumentException("The partitioer class must not be empty if you want to send to partition by partitioner.");
}
//if (producerConfigTemplate.TotalNumPartitions <= 0)
// throw new ArgumentException(string.Format("Please also specify the TotalNumPartitions if you want to send to partition by partitioner."));
ProduceByPartitionerClass(produceOptions, topicDataInZookeeper.Count);
}
else
{
ProduceToRandomOrSpecificPartition(produceOptions);
}
}
}
private static void PrepareSentMessages(ProduceSimpleHelperOption produceroundrobinOptions)
{
Console.WriteLine("start perf test, {0} ", DateTime.Now);
listOfDataNeedSendInOneBatch = new List <ProducerData <byte[], Message> >();
for (int i = 0; i < produceroundrobinOptions.MessageCountPerBatch; i++)
{
String val = KafkaClientHelperUtils.GetRandomString(produceroundrobinOptions.MessageSize);
byte[] bVal = System.Text.Encoding.UTF8.GetBytes(val);
byte[] bKey = System.Text.Encoding.UTF8.GetBytes(string.Format("{0:000000}", i));
if (produceroundrobinOptions.ConstantMessageKey)
{
bKey = System.Text.Encoding.UTF8.GetBytes(string.Format("{0:000000}", 0));
}
Message message = new Message(bVal, bKey, produceroundrobinOptions.CompressionCodec);
listOfKeys.Add(bKey);
listOfDataNeedSendInOneBatch.Add(new ProducerData <byte[], Message>(produceroundrobinOptions.Topic, bKey, message));
}
}
private static void ProduceToRandomOrSpecificPartition(ProduceSimpleHelperOption produceOptions)
{
using (KafkaSimpleManager <byte[], Message> kafkaSimpleManager = new KafkaSimpleManager <byte[], Message>(kafkaSimpleManagerConfig))
{
Stopwatch stopwatch2 = new Stopwatch();
stopwatch2.Start();
int nProducers = kafkaSimpleManager.InitializeProducerPoolForTopic(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true, producerConfigTemplate, false);
stopwatch2.Stop();
Console.WriteLine("Spent {0:0.00} ms to create {1} producers. average {2:0.00} ms per producer. syncProducerOfOnePartition:{3} ", stopwatch2.Elapsed.TotalMilliseconds, nProducers, stopwatch2.Elapsed.TotalMilliseconds / nProducers, producerConfigTemplate.SyncProducerOfOneBroker);
totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count();
int targetPartitionID = Int16.MinValue;
while (true)
{
Producer <byte[], Message> producer = null;
try
{
targetPartitionID = produceOptions.PartitionId >= 0 ? produceOptions.PartitionId : rand.Next(totalPartitionCount);
Logger.InfoFormat("Will try get producer for partition:{0} ", targetPartitionID);
producer = kafkaSimpleManager.GetProducerOfPartition(produceOptions.Topic, targetPartitionID, false);
Logger.InfoFormat("Get producer for partition:{0} Producer:{1} ", targetPartitionID, producer == null ? "null" : producer.ToString());
stopWatch.Start();
producer.Send(listOfDataNeedSendInOneBatch);
successMessageCount += produceOptions.MessageCountPerBatch;
if (!produceMessagePerPartition.ContainsKey(targetPartitionID))
{
produceMessagePerPartition.Add(targetPartitionID, 0);
}
produceMessagePerPartition[targetPartitionID] = produceMessagePerPartition[targetPartitionID] + listOfDataNeedSendInOneBatch.Count;
stopWatch.Stop();
sentBatchCount++;
if (sentBatchCount % 10 == 0)
{
RegularStatistics(produceOptions, sentBatchCount);
}
if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount)
{
break;
}
//Maybe partition increased, need regular check if partition number has changed.
if (DateTime.UtcNow > lastTimeRefreshMetadata.AddMinutes(10))
{
List <string> partitoins = kafkaSimpleManager.GetTopicPartitionsFromZK(produceOptions.Topic);
if (partitoins.Count != targetPartitionID)
{
nProducers = kafkaSimpleManager.InitializeProducerPoolForTopic(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true, producerConfigTemplate, false);
totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count();
targetPartitionID = produceOptions.PartitionId >= 0 ? produceOptions.PartitionId : rand.Next(totalPartitionCount);
Logger.InfoFormat("Will try get producer for partition:{0} ", targetPartitionID);
producer = kafkaSimpleManager.GetProducerOfPartition(produceOptions.Topic, targetPartitionID, false);
Logger.InfoFormat("Get producer for partition:{0} Producer:{1} ", targetPartitionID, producer == null ? "null" : producer.ToString());
}
lastTimeRefreshMetadata = DateTime.UtcNow;
}
}
catch (Exception e)
{
Logger.ErrorFormat("Got exception, maybe leader change or not available for some partition, will refresh metadata and recreate producer {0}", e.FormatException());
try
{
producer = kafkaSimpleManager.RefreshMetadataAndRecreateProducerOfOnePartition(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, targetPartitionID, true, true, producerConfigTemplate, false);
}
catch (Exception ex)
{
Logger.ErrorFormat("Got exception while RefreshMetadataAndRecreateProducerOfOnePartition, maybe leader change for some partition, will refresh metadata and recreate producer {0} after 3 seconds ...", ex.FormatException());
}
totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count();
if (targetPartitionID >= totalPartitionCount)
{
targetPartitionID = produceOptions.PartitionId >= 0 ? produceOptions.PartitionId : rand.Next(totalPartitionCount);
Logger.InfoFormat("Will try get producer for partition:{0} ", targetPartitionID);
producer = kafkaSimpleManager.GetProducerOfPartition(produceOptions.Topic, targetPartitionID, false);
Logger.InfoFormat("Get producer for partition:{0} Producer:{1} ", targetPartitionID, producer == null ? "null" : producer.ToString());
}
}
if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount)
{
break;
}
}
}
RegularStatistics(produceOptions, sentBatchCount);
}
private static void ProduceByPartitionerClass(ProduceSimpleHelperOption produceOptions, int partitionCountInZookeeper)
{
using (KafkaSimpleManager <byte[], Message> kafkaSimpleManager = new KafkaSimpleManager <byte[], Message>(kafkaSimpleManagerConfig))
{
Stopwatch stopwatch2 = new Stopwatch();
stopwatch2.Start();
int nProducers = kafkaSimpleManager.InitializeProducerPoolForTopic(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true, producerConfigTemplate, false);
stopwatch2.Stop();
Console.WriteLine("Spent {0:0.00} ms to create {1} producers. average {2:0.00} ms per producer. syncProducerOfOnePartition:{3} ", stopwatch2.Elapsed.TotalMilliseconds, nProducers, stopwatch2.Elapsed.TotalMilliseconds / nProducers, producerConfigTemplate.SyncProducerOfOneBroker);
while (true)
{
Producer <byte[], Message> producer = null;
try
{
producer = kafkaSimpleManager.GetProducerWithPartionerClass(produceOptions.Topic);
//Here the
totalPartitionCount = kafkaSimpleManager.GetTopicMetadta(produceOptions.Topic).PartitionsMetadata.Count();
Logger.InfoFormat("Get GetProducerWithPartionerClass Producer:{0} totalPartitionCount has metadata:{1} partitionCountInZookeeper:{2} "
, producer == null ? "null" : producer.ToString(), totalPartitionCount, partitionCountInZookeeper);
stopWatch.Start();
CountExpectCount(listOfKeys, produceOptions.PartitionerClass, partitionCountInZookeeper);
producer.Send(listOfDataNeedSendInOneBatch);
successMessageCount += produceOptions.MessageCountPerBatch;
stopWatch.Stop();
sentBatchCount++;
if (sentBatchCount % 10 == 0)
{
RegularStatistics(produceOptions, sentBatchCount);
}
if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount)
{
break;
}
}
catch (FailedToSendMessageException <byte[]> e)
{
//Sometime the sent maybe partially success.
//For example, 100 message, averagely to 5 partitions, if one partitoin has no leader.
// here will get the failed 20 message, you can retry or do something for them.
Logger.Error("===FAILED=========");
Logger.ErrorFormat("{0}", e.Message);
if (e.ProduceDispatchSeralizeResult != null &&
e.ProduceDispatchSeralizeResult.FailedProducerDatas != null)
{
Logger.ErrorFormat("Failed produce message key: ");
foreach (ProducerData <byte[], Message> a in e.ProduceDispatchSeralizeResult.FailedProducerDatas)
{
Logger.ErrorFormat("Key:{0} ", System.Text.Encoding.Default.GetString(a.Key));
}
}
//Here partially success we also consider
failedMessageCount += e.CountFailed;
successMessageCount += e.CountAll - e.CountFailed;
sentBatchCount++;
if (sentBatchCount % 10 == 0)
{
RegularStatistics(produceOptions, sentBatchCount);
}
if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount)
{
break;
}
Logger.Error(" \r\n");
}
catch (Exception e)
{
Logger.ErrorFormat("Got exception, maybe leader change for some partition, will refresh metadata and recreate producer {0}", e.FormatException());
TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(KafkaNETExampleConstants.DefaultVersionId, ClientID, ProducerRequestId++, produceOptions.Topic, true);
totalPartitionCount = topicMetadata.PartitionsMetadata.Count();
Logger.InfoFormat("Get GetProducerWithPartionerClass Producer:{0} totalPartitionCount has metadata:{1} partitionCountInZookeeper:{2} "
, producer == null ? "null" : producer.ToString(), totalPartitionCount, partitionCountInZookeeper);
}
if (produceOptions.BatchCount > 0 && sentBatchCount >= produceOptions.BatchCount)
{
break;
}
}
}
RegularStatistics(produceOptions, sentBatchCount);
}
public static void MainInternal(int taskIndexInStorm, string[] args)
{
GenerateAssistFile("producesimple");
GenerateAssistFile("produceperftest");
GenerateAssistFile("producemonitor");
GenerateAssistFile("eventserverperftest");
GenerateAssistFile("consumesimple");
GenerateAssistFile("consumegroup");
GenerateAssistFile("consumegroupmonitor");
GenerateAssistFile("topic");
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
var log4netSection = ConfigurationManager.GetSection("log4net");
if (log4netSection != null)
{
//XmlConfigurator.Configure();
}
KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb();
try
{
commandOptions.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage());
Environment.Exit(-1);
}
KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption;
try
{
realOption.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(realOption.GetUsage(false));
Environment.Exit(-1);
}
Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict());
switch (KafkaNETExampleCommandVerb.Verb)
{
case "producesimple":
case "produceroundrobin":
ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption;
ProduceSimpleHelper.Run(produceroundrobinOptions);
break;
case "produceperftest":
case "producewrapper":
ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption;
(new ProducePerfTestHelper()).Run(producewrapperOption);
break;
case "producem":
case "producemonitor":
ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption;
ProduceMonitorHelper.Run(produceMonitorOptions);
break;
case "eventserverperftest":
JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption;
(new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions);
break;
case "consumesimple":
case "dumpdata":
ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption;
ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions);
break;
case "consumegroup":
case "dumpdataasconsumergroup":
ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption;
if (taskIndexInStorm >= 0)
{
cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString();
cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File;
}
ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions);
break;
case "latestoffsetofconsumergroup":
case "consumegroupmonitor":
case "consumegroupm":
case "consumem":
ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption;
ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions);
break;
case "topic":
TopicHelperArguments dtOptions = (TopicHelperArguments)realOption;
TopicHelper.DumpTopicMetadataAndOffset(dtOptions);
break;
case "test":
var testOptions = (TestHelperOptions)realOption;
TestHelper.Run(testOptions);
break;
default:
Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb));
return;
}
}
internal void Parse(string[] args)
{
if (args == null || args.Length <= 0)
{
throw new ArgumentException("Please provide verb.");
}
Verb = args[0].ToLowerInvariant();
if ("topic" == Verb.ToLowerInvariant())
ActiveSubOption = new TopicHelperArguments();
else if ("dumpdata" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeDataHelperArguments();
else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupHelperOptions();
else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant()
|| "consumegroupm" == Verb.ToLowerInvariant()
|| "consumem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupMonitorHelperOptions();
else if ("produceroundrobin" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceSimpleHelperOption();
else if ("test" == Verb.ToLowerInvariant())
ActiveSubOption = new TestHelperOptions();
else if ("producewrapper" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProducePerfTestHelperOption();
else if ("producem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceMonitorHelperOptions();
else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new JavaEventServerPerfTestHelperOptions();
else
{
throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb));
}
}
