internal void Parse(string[] args)
{
if (args == null || args.Length <= 0)
{
throw new ArgumentException("Please provide verb.");
}
Verb = args[0].ToLowerInvariant();
if ("topic" == Verb.ToLowerInvariant())
ActiveSubOption = new TopicHelperArguments();
else if ("dumpdata" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeDataHelperArguments();
else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupHelperOptions();
else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant()
|| "consumegroupm" == Verb.ToLowerInvariant()
|| "consumem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupMonitorHelperOptions();
else if ("produceroundrobin" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceSimpleHelperOption();
else if ("test" == Verb.ToLowerInvariant())
ActiveSubOption = new TestHelperOptions();
else if ("producewrapper" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProducePerfTestHelperOption();
else if ("producem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceMonitorHelperOptions();
else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new JavaEventServerPerfTestHelperOptions();
else
{
throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb));
}
}
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions)
{
string topics = dtOptions.Topic;
string zookeeper = dtOptions.Zookeeper;
int partitionIndex = dtOptions.PartitionIndex;
bool includePartitionDetailInfo = true;
bool includeOffsetInfo = true;
DateTime timestamp = dtOptions.TimestampInUTC;
bool dumpToLog = true;
bool dumpToConsole = true;
string file = dtOptions.File;
using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
//BrokerID -->Count of as leader
SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>();
//BrokerID -->Count of as replica
SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>();
StringBuilder sbAll = new StringBuilder();
int topicCount = 0;
if (string.IsNullOrEmpty(topics))
{
List <string> topicList = new List <string>();
string path = "/brokers/topics";
try
{
IEnumerable <string> ts = zkClient.GetChildren(path);
foreach (var p in ts)
{
topicList.Add(p);
}
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e);
}
else
{
throw;
}
}
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count);
foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray())
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
topicCount = topicList.Count;
}
else if (topics.Contains(","))
{
string[] topicArray = topics.Split(new char[] { ',' });
topicCount = topicArray.Length;
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length);
foreach (var t in topicArray.OrderBy(r => r).ToArray())
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
}
else
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
topicCount = 1;
}
if (topicCount > 1)
{
sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n");
sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
foreach (KeyValuePair <int, int> kv in parttionBrokerID_LeaderCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n");
sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
foreach (KeyValuePair <int, int> kv in parttionBrokerID_ReplicaCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
}
string s = sbAll.ToString();
if (dumpToLog)
{
Logger.Info(s);
}
if (dumpToConsole)
{
Console.WriteLine(s);
}
if (!string.IsNullOrEmpty(file))
{
Console.WriteLine("Will write to {0}", file);
using (StreamWriter sw = new StreamWriter(file, false))
{
sw.WriteLine(s);
}
}
}
}
public static void MainInternal(int taskIndexInStorm, string[] args)
{
GenerateAssistFile("producesimple");
GenerateAssistFile("produceperftest");
GenerateAssistFile("producemonitor");
GenerateAssistFile("eventserverperftest");
GenerateAssistFile("consumesimple");
GenerateAssistFile("consumegroup");
GenerateAssistFile("consumegroupmonitor");
GenerateAssistFile("topic");
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
var log4netSection = ConfigurationManager.GetSection("log4net");
if (log4netSection != null)
{
//XmlConfigurator.Configure();
}
KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb();
try
{
commandOptions.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage());
Environment.Exit(-1);
}
KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption;
try
{
realOption.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(realOption.GetUsage(false));
Environment.Exit(-1);
}
Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict());
switch (KafkaNETExampleCommandVerb.Verb)
{
case "producesimple":
case "produceroundrobin":
ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption;
ProduceSimpleHelper.Run(produceroundrobinOptions);
break;
case "produceperftest":
case "producewrapper":
ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption;
(new ProducePerfTestHelper()).Run(producewrapperOption);
break;
case "producem":
case "producemonitor":
ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption;
ProduceMonitorHelper.Run(produceMonitorOptions);
break;
case "eventserverperftest":
JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption;
(new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions);
break;
case "consumesimple":
case "dumpdata":
ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption;
ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions);
break;
case "consumegroup":
case "dumpdataasconsumergroup":
ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption;
if (taskIndexInStorm >= 0)
{
cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString();
cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File;
}
ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions);
break;
case "latestoffsetofconsumergroup":
case "consumegroupmonitor":
case "consumegroupm":
case "consumem":
ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption;
ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions);
break;
case "topic":
TopicHelperArguments dtOptions = (TopicHelperArguments)realOption;
TopicHelper.DumpTopicMetadataAndOffset(dtOptions);
break;
case "test":
var testOptions = (TestHelperOptions)realOption;
TestHelper.Run(testOptions);
break;
default:
Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb));
return;
}
}
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions)
{
string topics = dtOptions.Topic;
string zookeeper = dtOptions.Zookeeper;
int partitionIndex = dtOptions.PartitionIndex;
bool includePartitionDetailInfo = true;
bool includeOffsetInfo = true;
DateTime timestamp = dtOptions.TimestampInUTC;
bool dumpToLog = true;
bool dumpToConsole = true;
string file = dtOptions.File;
using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
//BrokerID -->Count of as leader
SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
//BrokerID -->Count of as replica
SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
StringBuilder sbAll = new StringBuilder();
int topicCount = 0;
if (string.IsNullOrEmpty(topics))
{
List<string> topicList = new List<string>();
string path = "/brokers/topics";
try
{
IEnumerable<string> ts = zkClient.GetChildren(path);
foreach (var p in ts)
{
topicList.Add(p);
}
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e);
}
else
throw;
}
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count);
foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray())
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
topicCount = topicList.Count;
}
else if (topics.Contains(","))
{
string[] topicArray = topics.Split(new char[] { ',' });
topicCount = topicArray.Length;
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length);
foreach (var t in topicArray.OrderBy(r => r).ToArray())
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
}
else
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
topicCount = 1;
}
if (topicCount > 1)
{
sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n");
sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_LeaderCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n");
sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_ReplicaCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
}
string s = sbAll.ToString();
if (dumpToLog)
{
Logger.Info(s);
}
if (dumpToConsole)
{
Console.WriteLine(s);
}
if (!string.IsNullOrEmpty(file))
{
Console.WriteLine("Will write to {0}", file);
using (StreamWriter sw = new StreamWriter(file, false))
{
sw.WriteLine(s);
}
}
}
}
