internal void Parse(string[] args) { if (args == null || args.Length <= 0) { throw new ArgumentException("Please provide verb."); } Verb = args[0].ToLowerInvariant(); if ("topic" == Verb.ToLowerInvariant()) ActiveSubOption = new TopicHelperArguments(); else if ("dumpdata" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeDataHelperArguments(); else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupHelperOptions(); else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant() || "consumegroupm" == Verb.ToLowerInvariant() || "consumem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupMonitorHelperOptions(); else if ("produceroundrobin" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceSimpleHelperOption(); else if ("test" == Verb.ToLowerInvariant()) ActiveSubOption = new TestHelperOptions(); else if ("producewrapper" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProducePerfTestHelperOption(); else if ("producem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceMonitorHelperOptions(); else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new JavaEventServerPerfTestHelperOptions(); else { throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb)); } }
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions) { string topics = dtOptions.Topic; string zookeeper = dtOptions.Zookeeper; int partitionIndex = dtOptions.PartitionIndex; bool includePartitionDetailInfo = true; bool includeOffsetInfo = true; DateTime timestamp = dtOptions.TimestampInUTC; bool dumpToLog = true; bool dumpToConsole = true; string file = dtOptions.File; using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); //BrokerID -->Count of as leader SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>(); //BrokerID -->Count of as replica SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>(); StringBuilder sbAll = new StringBuilder(); int topicCount = 0; if (string.IsNullOrEmpty(topics)) { List <string> topicList = new List <string>(); string path = "/brokers/topics"; try { IEnumerable <string> ts = zkClient.GetChildren(path); foreach (var p in ts) { topicList.Add(p); } } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e); } else { throw; } } sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count); foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray()) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } topicCount = topicList.Count; } else if (topics.Contains(",")) { string[] topicArray = topics.Split(new char[] { ',' }); topicCount = topicArray.Length; sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length); foreach (var t in topicArray.OrderBy(r => r).ToArray()) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } } else { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); topicCount = 1; } if (topicCount > 1) { sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n"); sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n"); foreach (KeyValuePair <int, int> kv in parttionBrokerID_LeaderCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n"); sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n"); foreach (KeyValuePair <int, int> kv in parttionBrokerID_ReplicaCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } } string s = sbAll.ToString(); if (dumpToLog) { Logger.Info(s); } if (dumpToConsole) { Console.WriteLine(s); } if (!string.IsNullOrEmpty(file)) { Console.WriteLine("Will write to {0}", file); using (StreamWriter sw = new StreamWriter(file, false)) { sw.WriteLine(s); } } } }
public static void MainInternal(int taskIndexInStorm, string[] args) { GenerateAssistFile("producesimple"); GenerateAssistFile("produceperftest"); GenerateAssistFile("producemonitor"); GenerateAssistFile("eventserverperftest"); GenerateAssistFile("consumesimple"); GenerateAssistFile("consumegroup"); GenerateAssistFile("consumegroupmonitor"); GenerateAssistFile("topic"); ServicePointManager.DefaultConnectionLimit = 5000; ServicePointManager.UseNagleAlgorithm = false; var log4netSection = ConfigurationManager.GetSection("log4net"); if (log4netSection != null) { //XmlConfigurator.Configure(); } KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb(); try { commandOptions.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage()); Environment.Exit(-1); } KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption; try { realOption.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(realOption.GetUsage(false)); Environment.Exit(-1); } Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict()); switch (KafkaNETExampleCommandVerb.Verb) { case "producesimple": case "produceroundrobin": ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption; ProduceSimpleHelper.Run(produceroundrobinOptions); break; case "produceperftest": case "producewrapper": ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption; (new ProducePerfTestHelper()).Run(producewrapperOption); break; case "producem": case "producemonitor": ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption; ProduceMonitorHelper.Run(produceMonitorOptions); break; case "eventserverperftest": JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption; (new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions); break; case "consumesimple": case "dumpdata": ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption; ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions); break; case "consumegroup": case "dumpdataasconsumergroup": ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption; if (taskIndexInStorm >= 0) { cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString(); cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File; } ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions); break; case "latestoffsetofconsumergroup": case "consumegroupmonitor": case "consumegroupm": case "consumem": ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption; ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions); break; case "topic": TopicHelperArguments dtOptions = (TopicHelperArguments)realOption; TopicHelper.DumpTopicMetadataAndOffset(dtOptions); break; case "test": var testOptions = (TestHelperOptions)realOption; TestHelper.Run(testOptions); break; default: Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb)); return; } }
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions) { string topics = dtOptions.Topic; string zookeeper = dtOptions.Zookeeper; int partitionIndex = dtOptions.PartitionIndex; bool includePartitionDetailInfo = true; bool includeOffsetInfo = true; DateTime timestamp = dtOptions.TimestampInUTC; bool dumpToLog = true; bool dumpToConsole = true; string file = dtOptions.File; using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); //BrokerID -->Count of as leader SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>(); //BrokerID -->Count of as replica SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>(); StringBuilder sbAll = new StringBuilder(); int topicCount = 0; if (string.IsNullOrEmpty(topics)) { List<string> topicList = new List<string>(); string path = "/brokers/topics"; try { IEnumerable<string> ts = zkClient.GetChildren(path); foreach (var p in ts) { topicList.Add(p); } } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e); } else throw; } sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count); foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray()) { SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>(); SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } topicCount = topicList.Count; } else if (topics.Contains(",")) { string[] topicArray = topics.Split(new char[] { ',' }); topicCount = topicArray.Length; sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length); foreach (var t in topicArray.OrderBy(r => r).ToArray()) { SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>(); SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } } else { SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>(); SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); topicCount = 1; } if (topicCount > 1) { sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n"); sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n"); foreach (KeyValuePair<int, int> kv in parttionBrokerID_LeaderCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n"); sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n"); foreach (KeyValuePair<int, int> kv in parttionBrokerID_ReplicaCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } } string s = sbAll.ToString(); if (dumpToLog) { Logger.Info(s); } if (dumpToConsole) { Console.WriteLine(s); } if (!string.IsNullOrEmpty(file)) { Console.WriteLine("Will write to {0}", file); using (StreamWriter sw = new StreamWriter(file, false)) { sw.WriteLine(s); } } } }