internal static void DumpMessageAsConsumerGroup(ConsumeGroupHelperOptions cgOptions) { try { totalCount = 0; ZookeeperConsumerConnector.UseSharedStaticZookeeperClient = cgOptions.UseSharedStaticZookeeperClient; DumpMessageAsConsumerGroupSigleThreadBlock(cgOptions); Logger.InfoFormat("======TotallyRead:{0}=============", Interlocked.Read(ref totalCount)); Console.WriteLine("======TotallyRead:{0}=============", Interlocked.Read(ref totalCount)); Logger.InfoFormat("======New offset"); Console.WriteLine("======New offset"); long totalCountCommit = 0; foreach (var kv in newOffset.OrderBy(r => r.Key)) { string d = string.Format("Partition:{0}\t Old Offset:{1,10} --> New Offset:{2,10} Diff:{3}" , kv.Key, initialOffset == null || !initialOffset.ContainsKey(kv.Key) ? 0 : initialOffset[kv.Key], kv.Value, kv.Value - (initialOffset == null || !initialOffset.ContainsKey(kv.Key) ? 0 : initialOffset[kv.Key])); Logger.Info(d); Console.WriteLine(d); totalCountCommit += kv.Value - (initialOffset == null || !initialOffset.ContainsKey(kv.Key) ? 0 : initialOffset[kv.Key]); } //TODO: currently each partition maybe missed one message hasn't been commit. please refer kafka document:https://cwiki.apache.org/confluence/display/KAFKA/Compression //Hence, for compressed data, the consumed offset will be advanced one compressed message at a time. This has the side effect of possible duplicates in the event of a consumer failure. For uncompressed data, consumed offset will be advanced one message at a time. if (totalCountCommit != Interlocked.Read(ref totalCount)) { Logger.ErrorFormat("totalCountCommit {0} != totalCount {1}, check next line log see if it's reasonable:", totalCountCommit, Interlocked.Read(ref totalCount)); long diff = totalCountCommit - Interlocked.Read(ref totalCount); if (diff <= newOffset.Count && diff >= 0) { Logger.ErrorFormat(" the difference is reasonable, by design of kafkaNET.Library. For each partition, if not hit end of log, at least read one record from it!"); } else { Logger.ErrorFormat(" the difference is not reasonable ,please check log!"); } } else { Logger.InfoFormat("totalCountCommit {0} == totalCount {1}", totalCountCommit, Interlocked.Read(ref totalCount)); Console.WriteLine("totalCountCommit {0} == totalCount {1}", totalCountCommit, Interlocked.Read(ref totalCount)); } } catch (Exception ex) { Logger.ErrorFormat("Consumer group consume data. got exception:{0}\r\ninput parameter:Topic:{1}\tZookeeper:{2}\tconsumerGroupName:{3}\tconsumerId:{4}\tthreadCount:{5}\tcount:{6}" , ex.FormatException(), cgOptions.Topic, cgOptions.Zookeeper, cgOptions.ConsumerGroupName, cgOptions.ConsumerId, cgOptions.FetchThreadCountPerConsumer, cgOptions.Count); } }
internal static void DumpMessageAsConsumerGroup(ConsumeGroupHelperOptions cgOptions) { try { totalCount = 0; ZookeeperConsumerConnector.UseSharedStaticZookeeperClient = cgOptions.UseSharedStaticZookeeperClient; DumpMessageAsConsumerGroupSigleThreadBlock(cgOptions); Logger.InfoFormat("======TotallyRead:{0}=============", Interlocked.Read(ref totalCount)); Console.WriteLine("======TotallyRead:{0}=============", Interlocked.Read(ref totalCount)); Logger.InfoFormat("======New offset"); Console.WriteLine("======New offset"); long totalCountCommit = 0; foreach (var kv in newOffset.OrderBy(r => r.Key)) { string d = string.Format("Partition:{0}\t Old Offset:{1,10} --> New Offset:{2,10} Diff:{3}" , kv.Key, initialOffset == null || !initialOffset.ContainsKey(kv.Key) ? 0 : initialOffset[kv.Key], kv.Value, kv.Value - (initialOffset == null || !initialOffset.ContainsKey(kv.Key) ? 0 : initialOffset[kv.Key])); Logger.Info(d); Console.WriteLine(d); totalCountCommit += kv.Value - (initialOffset == null || !initialOffset.ContainsKey(kv.Key) ? 0 : initialOffset[kv.Key]); } //TODO: currently each partition maybe missed one message hasn't been commit. please refer kafka document:https://cwiki.apache.org/confluence/display/KAFKA/Compression //Hence, for compressed data, the consumed offset will be advanced one compressed message at a time. This has the side effect of possible duplicates in the event of a consumer failure. For uncompressed data, consumed offset will be advanced one message at a time. if (totalCountCommit != Interlocked.Read(ref totalCount)) { Logger.ErrorFormat("totalCountCommit {0} != totalCount {1}, check next line log see if it's reasonable:", totalCountCommit, Interlocked.Read(ref totalCount)); long diff = totalCountCommit - Interlocked.Read(ref totalCount); if (diff <= newOffset.Count && diff >= 0) { Logger.ErrorFormat(" the difference is reasonable, by design of kafkaNET.Library. For each partition, if not hit end of log, at least read one record from it!"); } else { Logger.ErrorFormat(" the difference is not reasonable ,please check log!"); } } else { Logger.InfoFormat("totalCountCommit {0} == totalCount {1}", totalCountCommit, Interlocked.Read(ref totalCount)); Console.WriteLine("totalCountCommit {0} == totalCount {1}", totalCountCommit, Interlocked.Read(ref totalCount)); } } catch (Exception ex) { Logger.ErrorFormat("Consumer group consume data. got exception:{0}\r\ninput parameter:Topic:{1}\tZookeeper:{2}\tconsumerGroupName:{3}\tconsumerId:{4}\tthreadCount:{5}\tcount:{6}" , ex.FormatException(), cgOptions.Topic, cgOptions.Zookeeper, cgOptions.ConsumerGroupName, cgOptions.ConsumerId, cgOptions.FetchThreadCountPerConsumer, cgOptions.Count); } }
//#region Option 1, single ThreadBlock private static void DumpMessageAsConsumerGroupSigleThreadBlock(ConsumeGroupHelperOptions cgOptions) { initialOffset = null; newOffset = null; AutoResetEvent[] autos = new AutoResetEvent[cgOptions.ZookeeperConnectorCount]; for (int i = 0; i < cgOptions.ZookeeperConnectorCount; i++) { AutoResetEvent resetEvent = new AutoResetEvent(false); ConsumerGroupHelperUnit unit = new ConsumerGroupHelperUnit(DateTime.UtcNow.Second, cgOptions, resetEvent, cgOptions.ZookeeperConnectorConsumeMessageCount[i]); Thread t = new Thread(unit.Consume); t.Start(); Logger.InfoFormat("Start thread {0} of ZookeeperConsumerConnector", i); autos[i] = resetEvent; } WaitHandle.WaitAll(autos); }
internal void Parse(string[] args) { if (args == null || args.Length <= 0) { throw new ArgumentException("Please provide verb."); } Verb = args[0].ToLowerInvariant(); if ("topic" == Verb.ToLowerInvariant()) ActiveSubOption = new TopicHelperArguments(); else if ("dumpdata" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeDataHelperArguments(); else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupHelperOptions(); else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant() || "consumegroupm" == Verb.ToLowerInvariant() || "consumem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupMonitorHelperOptions(); else if ("produceroundrobin" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceSimpleHelperOption(); else if ("test" == Verb.ToLowerInvariant()) ActiveSubOption = new TestHelperOptions(); else if ("producewrapper" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProducePerfTestHelperOption(); else if ("producem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceMonitorHelperOptions(); else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new JavaEventServerPerfTestHelperOptions(); else { throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb)); } }
internal ConsumerGroupHelperUnit(int threadID, ConsumeGroupHelperOptions cg, AutoResetEvent e, int c) { this.resetEvent = e; this.cgOptions = cg; this.ThreadID = threadID; this.Count = c; configSettings = new ConsumerConfiguration { AutoOffsetReset = OffsetRequest.SmallestTime, AutoCommit = false, GroupId = cgOptions.ConsumerGroupName, ConsumerId = cgOptions.ConsumerId + "_Thread_" + threadID.ToString(), Timeout = cgOptions.Timeout, ZooKeeper = new ZooKeeperConfiguration(cgOptions.Zookeeper, 30000, 4000, 8000), BufferSize = cgOptions.BufferSize, FetchSize = cgOptions.FetchSize, MaxFetchBufferLength = cgOptions.MaxFetchBufferLength// cgOptions.FetchSize * (10~40) / cgOptions.MessageSize, }; if (cgOptions.CancellationTimeoutMs != 5000) { configSettings.Timeout = -1; } }
//#region Option 1, single ThreadBlock private static void DumpMessageAsConsumerGroupSigleThreadBlock(ConsumeGroupHelperOptions cgOptions) { initialOffset = null; newOffset = null; AutoResetEvent[] autos = new AutoResetEvent[cgOptions.ZookeeperConnectorCount]; for (int i = 0; i < cgOptions.ZookeeperConnectorCount; i++) { AutoResetEvent resetEvent = new AutoResetEvent(false); ConsumerGroupHelperUnit unit = new ConsumerGroupHelperUnit(i, cgOptions, resetEvent, cgOptions.ZookeeperConnectorConsumeMessageCount[i]); Thread t = new Thread(unit.Consume); t.Start(); Logger.InfoFormat("Start thread {0} of ZookeeperConsumerConnector", i); autos[i] = resetEvent; } WaitHandle.WaitAll(autos); }
internal ConsumerGroupHelperUnit(int threadID, ConsumeGroupHelperOptions cg, AutoResetEvent e, int c) { this.resetEvent = e; this.cgOptions = cg; this.ThreadID = threadID; this.Count = c; configSettings = new ConsumerConfiguration { AutoOffsetReset = OffsetRequest.SmallestTime, AutoCommit = false, GroupId = cgOptions.ConsumerGroupName, ConsumerId = cgOptions.ConsumerId + "_Thread_" + threadID.ToString(), Timeout = cgOptions.Timeout, ZooKeeper = new ZooKeeperConfiguration(cgOptions.Zookeeper, 30000, 4000, 8000), BufferSize = cgOptions.BufferSize, FetchSize = cgOptions.FetchSize, MaxFetchBufferLength = cgOptions.MaxFetchBufferLength// cgOptions.FetchSize * (10~40) / cgOptions.MessageSize, }; if (cgOptions.CancellationTimeoutMs != KafkaNETExampleConstants.DefaultCancellationTimeoutMs) configSettings.Timeout = -1; }
public static void MainInternal(int taskIndexInStorm, string[] args) { GenerateAssistFile("producesimple"); GenerateAssistFile("produceperftest"); GenerateAssistFile("producemonitor"); GenerateAssistFile("eventserverperftest"); GenerateAssistFile("consumesimple"); GenerateAssistFile("consumegroup"); GenerateAssistFile("consumegroupmonitor"); GenerateAssistFile("topic"); ServicePointManager.DefaultConnectionLimit = 5000; ServicePointManager.UseNagleAlgorithm = false; var log4netSection = ConfigurationManager.GetSection("log4net"); if (log4netSection != null) { //XmlConfigurator.Configure(); } KafkaNETExampleCommandVerb commandOptions = new KafkaNETExampleCommandVerb(); try { commandOptions.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(KafkaNETExampleCommandVerb.GetUsage()); Environment.Exit(-1); } KafkaNETExampleSubArguments realOption = KafkaNETExampleCommandVerb.ActiveSubOption; try { realOption.Parse(args); } catch (Exception e) { Logger.ErrorFormat("{0}", e.FormatException()); Console.WriteLine(realOption.GetUsage(false)); Environment.Exit(-1); } Logger.InfoFormat("All arguments of {0}: \r\n{1}", KafkaNETExampleCommandVerb.AssemblyName, realOption.GetArgDict()); switch (KafkaNETExampleCommandVerb.Verb) { case "producesimple": case "produceroundrobin": ProduceSimpleHelperOption produceroundrobinOptions = (ProduceSimpleHelperOption)realOption; ProduceSimpleHelper.Run(produceroundrobinOptions); break; case "produceperftest": case "producewrapper": ProducePerfTestHelperOption producewrapperOption = (ProducePerfTestHelperOption)realOption; (new ProducePerfTestHelper()).Run(producewrapperOption); break; case "producem": case "producemonitor": ProduceMonitorHelperOptions produceMonitorOptions = (ProduceMonitorHelperOptions)realOption; ProduceMonitorHelper.Run(produceMonitorOptions); break; case "eventserverperftest": JavaEventServerPerfTestHelperOptions evetServerPerfTestOptions = (JavaEventServerPerfTestHelperOptions)realOption; (new JavaEventServerPerfTestHelper()).Run(evetServerPerfTestOptions); break; case "consumesimple": case "dumpdata": ConsumeDataHelperArguments dumpdataOptions = (ConsumeDataHelperArguments)realOption; ConsumeSimpleHelper.ConsumeDataSimple(dumpdataOptions); break; case "consumegroup": case "dumpdataasconsumergroup": ConsumeGroupHelperOptions cgOptions = (ConsumeGroupHelperOptions)realOption; if (taskIndexInStorm >= 0) { cgOptions.ConsumerId = cgOptions.ConsumerId + taskIndexInStorm.ToString(); cgOptions.File = cgOptions.ConsumerId + taskIndexInStorm.ToString() + cgOptions.File; } ConsumerGroupHelper.DumpMessageAsConsumerGroup(cgOptions); break; case "latestoffsetofconsumergroup": case "consumegroupmonitor": case "consumegroupm": case "consumem": ConsumeGroupMonitorHelperOptions dcgOptions = (ConsumeGroupMonitorHelperOptions)realOption; ConsumeGroupMonitorHelper.DumpConsumerGroupOffsets(dcgOptions); break; case "topic": TopicHelperArguments dtOptions = (TopicHelperArguments)realOption; TopicHelper.DumpTopicMetadataAndOffset(dtOptions); break; case "test": var testOptions = (TestHelperOptions)realOption; TestHelper.Run(testOptions); break; default: Logger.Error(string.Format("Invalid verb={0}", KafkaNETExampleCommandVerb.Verb)); return; } }