private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List <ConsumeGroupMonitorUnit> units) { Logger.Info("Will refresh all consume group from zk ..."); string path = "/consumers"; //TODO: add /users IEnumerable <string> groups = zkClient.GetChildren(path); foreach (var g in groups) { Logger.InfoFormat("Will check group {0}", g); string topicpath = string.Format("/consumers/{0}/owners", g); IEnumerable <string> topics = zkClient.GetChildren(topicpath); foreach (var t in topics) { Logger.InfoFormat("Will check topic {0}", t); units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t)); } } Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count); }
internal void Parse(string[] args) { if (args == null || args.Length <= 0) { throw new ArgumentException("Please provide verb."); } Verb = args[0].ToLowerInvariant(); if ("topic" == Verb.ToLowerInvariant()) ActiveSubOption = new TopicHelperArguments(); else if ("dumpdata" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeDataHelperArguments(); else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupHelperOptions(); else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant() || "consumegroupm" == Verb.ToLowerInvariant() || "consumem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupMonitorHelperOptions(); else if ("produceroundrobin" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceSimpleHelperOption(); else if ("test" == Verb.ToLowerInvariant()) ActiveSubOption = new TestHelperOptions(); else if ("producewrapper" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProducePerfTestHelperOption(); else if ("producem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceMonitorHelperOptions(); else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new JavaEventServerPerfTestHelperOptions(); else { throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb)); } }
internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption) { List <ConsumeGroupMonitorUnit> units = new List <ConsumeGroupMonitorUnit>(); if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray)) { if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName)) { units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic)); } else { danymicAllConsumeGroupTopic = true; } } else { string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); foreach (var cg in cgs) { string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); if (temp.Length != 2) { Logger.ErrorFormat("Wrong parameter, Exmaple G1:t1,G2:t2. The value:{0} is invalid.", cg); } else { units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1])); } } } long count = 0; int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds; using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); while (true) { DateTime time = DateTime.Now; string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day); if (!Directory.Exists(dateFolder)) { Directory.CreateDirectory(dateFolder); } if (danymicAllConsumeGroupTopic && count % cycle == 0) { units = new List <ConsumeGroupMonitorUnit>(); RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units); } int countSuccess = 0; foreach (var unit in units) { unit.subFolder = dateFolder; if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper)) { countSuccess++; } } if (countSuccess == units.Count) { Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess); } else { Logger.ErrorFormat("===========All {0} consume group only {1} succ. FAIL=================", units.Count, countSuccess); } Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000); count++; } } }
internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption) { List<ConsumeGroupMonitorUnit> units = new List<ConsumeGroupMonitorUnit>(); if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray)) { if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName)) units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic)); else { danymicAllConsumeGroupTopic = true; } } else { string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); foreach (var cg in cgs) { string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); if (temp.Length != 2) { Logger.ErrorFormat("Wrong parameter, Exmaple G1:t1,G2:t2. The value:{0} is invalid.", cg); } else units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1])); } } long count = 0; int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds; using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); while (true) { DateTime time = DateTime.Now; string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day); if (!Directory.Exists(dateFolder)) Directory.CreateDirectory(dateFolder); if (danymicAllConsumeGroupTopic && count % cycle == 0) { units = new List<ConsumeGroupMonitorUnit>(); RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units); } int countSuccess = 0; foreach (var unit in units) { unit.subFolder = dateFolder; if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper)) countSuccess++; } if (countSuccess == units.Count) Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess); else Logger.ErrorFormat("===========All {0} consume group only {1} succ. FAIL=================", units.Count, countSuccess); Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000); count++; } } }
private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List<ConsumeGroupMonitorUnit> units) { Logger.Info("Will refresh all consume group from zk ..."); string path = "/consumers"; //TODO: add /users IEnumerable<string> groups = zkClient.GetChildren(path); foreach (var g in groups) { Logger.InfoFormat("Will check group {0}", g); string topicpath = string.Format("/consumers/{0}/owners", g); IEnumerable<string> topics = zkClient.GetChildren(topicpath); foreach (var t in topics) { Logger.InfoFormat("Will check topic {0}", t); units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t)); } } Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count); }
public static void MainInternal(int taskIndexInStorm, string[] args) { GenerateAssistFile("producesimple"); GenerateAssistFile("produceperftest"); GenerateAssistFile("producemonitor"); GenerateAssistFile("eventserverperftest"); GenerateAssistFile("consumesimple"); GenerateAssistFile("consumegroup"); GenerateAssistFile("consumegroupmonitor"); GenerateAssistFile("topic"); ServicePointManager.DefaultConnectionLimit = 5000; ServicePointManager.UseNagleAlgorithm = false; var log4netSection = ConfigurationManager.GetSection("log4net"); if (log4netSection != null) { //XmlConfigurator.Configure(); } KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb(); try { commandOptions.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage()); Environment.Exit(-1); } KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption; try { realOption.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(realOption.GetUsage(false)); Environment.Exit(-1); } Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict()); switch (KafkaNETExampleCommandVerb.Verb) { case "producesimple": case "produceroundrobin": ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption; ProduceSimpleHelper.Run(produceroundrobinOptions); break; case "produceperftest": case "producewrapper": ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption; (new ProducePerfTestHelper()).Run(producewrapperOption); break; case "producem": case "producemonitor": ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption; ProduceMonitorHelper.Run(produceMonitorOptions); break; case "eventserverperftest": JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption; (new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions); break; case "consumesimple": case "dumpdata": ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption; ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions); break; case "consumegroup": case "dumpdataasconsumergroup": ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption; if (taskIndexInStorm >= 0) { cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString(); cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File; } ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions); break; case "latestoffsetofconsumergroup": case "consumegroupmonitor": case "consumegroupm": case "consumem": ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption; ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions); break; case "topic": TopicHelperArguments dtOptions = (TopicHelperArguments)realOption; TopicHelper.DumpTopicMetadataAndOffset(dtOptions); break; case "test": var testOptions = (TestHelperOptions)realOption; TestHelper.Run(testOptions); break; default: Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb)); return; } }