private static void Initializer(ProducePerfTestHelperOption producewrapperOption)
{
Logger.InfoFormat("prepare perf test, {0} ", DateTime.Now);
listOfDataNeedSendInOneBatch = new List <ProducerData <byte[], Message> >();
for (int i = 0; i < producewrapperOption.MessageCountPerBatch; i++)
{
String vKey = KafkaClientHelperUtils.GetRandomString(32);
byte[] bKey = System.Text.Encoding.UTF8.GetBytes(vKey);
String val = KafkaClientHelperUtils.GetRandomString(producewrapperOption.MessageSize);
byte[] bVal = System.Text.Encoding.UTF8.GetBytes(val);
Message message = new Message(bVal, bKey, producewrapperOption.CompressionCodec);
listOfDataNeedSendInOneBatch.Add(new ProducerData <byte[], Message>(producewrapperOption.Topic, message));
}
}
internal void Run(ProducePerfTestHelperOption option)
{
produceOption = option;
ProducePerfTestKafkaSimpleManagerWrapper.produceOptions = option;
Initializer(option);
Thread[] threads = new Thread[option.ThreadCount];
AutoResetEvent[] autos = new AutoResetEvent[option.ThreadCount];
threadPareamaters = new PerfTestThreadParameter[option.ThreadCount];
Logger.InfoFormat("start send {0} ", DateTime.Now);
stopWatch.Restart();
//[('p', -1 random partition. -2 by default or customized partitioner class. (you need specify PartitionerClass by -l) >=0 to specific partition.
for (int i = 0; i < option.ThreadCount; i++)
{
AutoResetEvent Auto = new AutoResetEvent(false);
PerfTestThreadParameter p = new PerfTestThreadParameter();
p.ThreadId = i;
p.SpeedConstrolMBPerSecond = (option.SpeedConstrolMBPerSecond * 1.0) / option.ThreadCount;
p.EventOfFinish = Auto;
if (produceOption.PartitionId < 0)
{
p.RandomReturnIfNotExist = true;
}
else
{
p.RandomReturnIfNotExist = false;
}
p.PartitionId = produceOption.PartitionId;
p.Topic = produceOption.Topic;
Logger.InfoFormat("Thread:{0} PartitionID:{1} RandomReturnIfNotExist:{2} ", p.ThreadId, p.PartitionId, p.RandomReturnIfNotExist);
Thread t = new Thread(new ParameterizedThreadStart(RunOneThread));
t.Start(p);
autos[i] = Auto;
threadPareamaters[i] = p;
}
WaitHandle.WaitAll(autos);
stopWatch.Stop();
Statistics();
}
public static void MainInternal(int taskIndexInStorm, string[] args)
{
GenerateAssistFile("producesimple");
GenerateAssistFile("produceperftest");
GenerateAssistFile("producemonitor");
GenerateAssistFile("eventserverperftest");
GenerateAssistFile("consumesimple");
GenerateAssistFile("consumegroup");
GenerateAssistFile("consumegroupmonitor");
GenerateAssistFile("topic");
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
var log4netSection = ConfigurationManager.GetSection("log4net");
if (log4netSection != null)
{
//XmlConfigurator.Configure();
}
KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb();
try
{
commandOptions.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage());
Environment.Exit(-1);
}
KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption;
try
{
realOption.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(realOption.GetUsage(false));
Environment.Exit(-1);
}
Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict());
switch (KafkaNETExampleCommandVerb.Verb)
{
case "producesimple":
case "produceroundrobin":
ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption;
ProduceSimpleHelper.Run(produceroundrobinOptions);
break;
case "produceperftest":
case "producewrapper":
ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption;
(new ProducePerfTestHelper()).Run(producewrapperOption);
break;
case "producem":
case "producemonitor":
ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption;
ProduceMonitorHelper.Run(produceMonitorOptions);
break;
case "eventserverperftest":
JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption;
(new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions);
break;
case "consumesimple":
case "dumpdata":
ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption;
ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions);
break;
case "consumegroup":
case "dumpdataasconsumergroup":
ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption;
if (taskIndexInStorm >= 0)
{
cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString();
cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File;
}
ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions);
break;
case "latestoffsetofconsumergroup":
case "consumegroupmonitor":
case "consumegroupm":
case "consumem":
ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption;
ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions);
break;
case "topic":
TopicHelperArguments dtOptions = (TopicHelperArguments)realOption;
TopicHelper.DumpTopicMetadataAndOffset(dtOptions);
break;
case "test":
var testOptions = (TestHelperOptions)realOption;
TestHelper.Run(testOptions);
break;
default:
Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb));
return;
}
}