private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List <ConsumeGroupMonitorUnit> units)
{
Logger.Info("Will refresh all consume group from zk ...");
string path = "/consumers";
//TODO: add /users
IEnumerable <string> groups = zkClient.GetChildren(path);
foreach (var g in groups)
{
Logger.InfoFormat("Will check group {0}", g);
string topicpath = string.Format("/consumers/{0}/owners", g);
IEnumerable <string> topics = zkClient.GetChildren(topicpath);
foreach (var t in topics)
{
Logger.InfoFormat("Will check topic {0}", t);
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t));
}
}
Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count);
}
internal void Parse(string[] args)
{
if (args == null || args.Length <= 0)
{
throw new ArgumentException("Please provide verb.");
}
Verb = args[0].ToLowerInvariant();
if ("topic" == Verb.ToLowerInvariant())
ActiveSubOption = new TopicHelperArguments();
else if ("dumpdata" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeDataHelperArguments();
else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupHelperOptions();
else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant()
|| "consumegroupm" == Verb.ToLowerInvariant()
|| "consumem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupMonitorHelperOptions();
else if ("produceroundrobin" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceSimpleHelperOption();
else if ("test" == Verb.ToLowerInvariant())
ActiveSubOption = new TestHelperOptions();
else if ("producewrapper" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProducePerfTestHelperOption();
else if ("producem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceMonitorHelperOptions();
else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new JavaEventServerPerfTestHelperOptions();
else
{
throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb));
}
}
internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption)
{
List <ConsumeGroupMonitorUnit> units = new List <ConsumeGroupMonitorUnit>();
if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray))
{
if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName))
{
units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic));
}
else
{
danymicAllConsumeGroupTopic = true;
}
}
else
{
string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var cg in cgs)
{
string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
if (temp.Length != 2)
{
Logger.ErrorFormat("Wrong parameter, Exmaple G1:t1,G2:t2. The value:{0} is invalid.", cg);
}
else
{
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1]));
}
}
}
long count = 0;
int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds;
using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
while (true)
{
DateTime time = DateTime.Now;
string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day);
if (!Directory.Exists(dateFolder))
{
Directory.CreateDirectory(dateFolder);
}
if (danymicAllConsumeGroupTopic && count % cycle == 0)
{
units = new List <ConsumeGroupMonitorUnit>();
RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units);
}
int countSuccess = 0;
foreach (var unit in units)
{
unit.subFolder = dateFolder;
if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper))
{
countSuccess++;
}
}
if (countSuccess == units.Count)
{
Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess);
}
else
{
Logger.ErrorFormat("===========All {0} consume group only {1} succ. FAIL=================", units.Count, countSuccess);
}
Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000);
count++;
}
}
}
internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption)
{
List<ConsumeGroupMonitorUnit> units = new List<ConsumeGroupMonitorUnit>();
if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray))
{
if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName))
units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic));
else
{
danymicAllConsumeGroupTopic = true;
}
}
else
{
string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var cg in cgs)
{
string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
if (temp.Length != 2)
{
Logger.ErrorFormat("Wrong parameter, Exmaple G1:t1,G2:t2. The value:{0} is invalid.", cg);
}
else
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1]));
}
}
long count = 0;
int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds;
using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
while (true)
{
DateTime time = DateTime.Now;
string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day);
if (!Directory.Exists(dateFolder))
Directory.CreateDirectory(dateFolder);
if (danymicAllConsumeGroupTopic && count % cycle == 0)
{
units = new List<ConsumeGroupMonitorUnit>();
RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units);
}
int countSuccess = 0;
foreach (var unit in units)
{
unit.subFolder = dateFolder;
if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper))
countSuccess++;
}
if (countSuccess == units.Count)
Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess);
else
Logger.ErrorFormat("===========All {0} consume group only {1} succ. FAIL=================", units.Count, countSuccess);
Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000);
count++;
}
}
}
private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List<ConsumeGroupMonitorUnit> units)
{
Logger.Info("Will refresh all consume group from zk ...");
string path = "/consumers";
//TODO: add /users
IEnumerable<string> groups = zkClient.GetChildren(path);
foreach (var g in groups)
{
Logger.InfoFormat("Will check group {0}", g);
string topicpath = string.Format("/consumers/{0}/owners", g);
IEnumerable<string> topics = zkClient.GetChildren(topicpath);
foreach (var t in topics)
{
Logger.InfoFormat("Will check topic {0}", t);
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t));
}
}
Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count);
}
public static void MainInternal(int taskIndexInStorm, string[] args)
{
GenerateAssistFile("producesimple");
GenerateAssistFile("produceperftest");
GenerateAssistFile("producemonitor");
GenerateAssistFile("eventserverperftest");
GenerateAssistFile("consumesimple");
GenerateAssistFile("consumegroup");
GenerateAssistFile("consumegroupmonitor");
GenerateAssistFile("topic");
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
var log4netSection = ConfigurationManager.GetSection("log4net");
if (log4netSection != null)
{
//XmlConfigurator.Configure();
}
KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb();
try
{
commandOptions.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage());
Environment.Exit(-1);
}
KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption;
try
{
realOption.Parse(args);
}
catch (Exception e)
{
Logger.ErrorFormat("{0}", e.FormatException());
Console.WriteLine(realOption.GetUsage(false));
Environment.Exit(-1);
}
Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict());
switch (KafkaNETExampleCommandVerb.Verb)
{
case "producesimple":
case "produceroundrobin":
ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption;
ProduceSimpleHelper.Run(produceroundrobinOptions);
break;
case "produceperftest":
case "producewrapper":
ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption;
(new ProducePerfTestHelper()).Run(producewrapperOption);
break;
case "producem":
case "producemonitor":
ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption;
ProduceMonitorHelper.Run(produceMonitorOptions);
break;
case "eventserverperftest":
JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption;
(new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions);
break;
case "consumesimple":
case "dumpdata":
ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption;
ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions);
break;
case "consumegroup":
case "dumpdataasconsumergroup":
ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption;
if (taskIndexInStorm >= 0)
{
cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString();
cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File;
}
ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions);
break;
case "latestoffsetofconsumergroup":
case "consumegroupmonitor":
case "consumegroupm":
case "consumem":
ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption;
ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions);
break;
case "topic":
TopicHelperArguments dtOptions = (TopicHelperArguments)realOption;
TopicHelper.DumpTopicMetadataAndOffset(dtOptions);
break;
case "test":
var testOptions = (TestHelperOptions)realOption;
TestHelper.Run(testOptions);
break;
default:
Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb));
return;
}
}