private static bool TestBug1490652Verify(TestHelperOptions testOptions) { bool verifyResult = true; foreach (KeyValuePair <int, Dictionary <int, string> > kv in TestBug1490652DataSent) { if (!TestBug1490652DataRead.ContainsKey(kv.Key)) { verifyResult = false; Logger.ErrorFormat("Result doesn't contains data for partition {0} ", kv.Key); } else { Dictionary <int, string> expectedResultOfOnePartition = kv.Value; Dictionary <int, string> resultOfOnePartition = TestBug1490652DataRead[kv.Key]; foreach (KeyValuePair <int, string> kv2 in expectedResultOfOnePartition) { if (resultOfOnePartition[kv2.Key] != kv2.Value) { verifyResult = false; Logger.ErrorFormat("Result doesn't match for partition {0} offset {1} data {2} vs {3} ", kv.Key, kv2.Key, kv2.Value, resultOfOnePartition[kv2.Key]); } } } } return(verifyResult); }
private static void TestBug1490652ReadData(TestHelperOptions testOptions) { KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration() { FetchSize = KafkaSimpleManagerConfiguration.DefaultFetchSize, BufferSize = KafkaSimpleManagerConfiguration.DefaultBufferSize, MaxWaitTime = 0, MinWaitBytes = 0, Zookeeper = testOptions.Zookeeper }; config.Verify(); using (KafkaSimpleManager <int, Message> kafkaSimpleManager = new KafkaSimpleManager <int, Message>(config)) { TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(0, "ClientID", 0, testOptions.Topic, true); PartitionCount = topicMetadata.PartitionsMetadata.Count(); for (int i = 0; i < PartitionCount; i++) { #region Get real offset and adjust long earliest = 0; long latest = 0; long offsetBase = 0; OffsetHelper.GetAdjustedOffset <int, Message>(testOptions.Topic, kafkaSimpleManager, i, KafkaOffsetType.Earliest, 0, 0, out earliest, out latest, out offsetBase); #endregion TestBug1490652DataRead.Add(i, ConsumeDataOfOnePartitionTotally <int, Message>(testOptions.Topic, kafkaSimpleManager, i, KafkaOffsetType.Earliest, 0, 0, latest, 0, 100, -1, "DumpLog.log")); } } }
private static void TestBug1490652SendData(TestHelperOptions testOptions) { int correlationID = 0; Random rand = new Random(); StringBuilder sb = new StringBuilder(); try { KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration() { Zookeeper = testOptions.Zookeeper, MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize }; config.Verify(); using (KafkaSimpleManager <int, Kafka.Client.Messages.Message> kafkaSimpleManager = new KafkaSimpleManager <int, Kafka.Client.Messages.Message>(config)) { TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(0, "ClientID", correlationID++, testOptions.Topic, true); PartitionCount = topicMetadata.PartitionsMetadata.Count(); List <ProducerData <int, Message> > listOfDataNeedSendInOneBatch = new List <ProducerData <int, Message> >(); for (int i = 0; i < PartitionCount; i++) { TestBug1490652DataSent.Add(i, new Dictionary <int, string>()); for (int j = 0; j < TestBug1490652MessageCountPerPartition; j++) { string val = KafkaClientHelperUtils.GetRandomString(testOptions.MessageSize); byte[] bVal = System.Text.Encoding.UTF8.GetBytes(val); //Set the key to partitionID, so it can directly fall into that partition. Message message = new Message(bVal, CompressionCodecs.DefaultCompressionCodec); listOfDataNeedSendInOneBatch.Add(new ProducerData <int, Message>(testOptions.Topic, i, message)); TestBug1490652DataSent[i].Add(j, val); } } ProducerConfiguration producerConfig = new ProducerConfiguration(new List <BrokerConfiguration>() { }) { PartitionerClass = ProducerConfiguration.DefaultPartitioner, RequiredAcks = 1, BufferSize = config.BufferSize, ZooKeeper = config.ZookeeperConfig, MaxMessageSize = Math.Max(config.MaxMessageSize, Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, testOptions.MessageSize)) }; producerConfig.SyncProducerOfOneBroker = 1; Producer <int, Kafka.Client.Messages.Message> producer = new Producer <int, Kafka.Client.Messages.Message>(producerConfig); producer.Send(listOfDataNeedSendInOneBatch); } } catch (Exception ex) { Logger.ErrorFormat("Produce data Got exception:{0}\r\ninput parameter: {1}\r\n" , ex.FormatException(), testOptions.ToString()); } }
/// <summary> /// Run one test case /// </summary> /// <param name="testOptions"></param> public static void Run(TestHelperOptions testOptions) { bool testResult = true; if (testOptions.Case.ToLowerInvariant() == "bug1490652") { testResult &= TestBug1490652(testOptions); } if (testResult) { Console.WriteLine("PASS"); } else { Logger.Error("Test cases FAILED!"); } }
/// <summary> /// Previously , the wrong exception hit when: /// One broker is leader of multiple partition(>=2). /// Send a group of message, and they need go to different partitions of the same broker. /// Preparation before run the case: /// Assume we have 3 broker, then create topic as /// .\bin\kafka.cmd topiccmd --create --topic mvlogsA --partition 4 --replication-factor 1 --zookeeper localhost /// Then run this case by send more than 5 messages. /// </summary> /// <param name="testOptions"></param> /// <returns></returns> private static bool TestBug1490652(TestHelperOptions testOptions) { try { //Send data TestBug1490652SendData(testOptions); DumpDictToFile(TestBug1490652DataSent, "Sent.log"); //Read data TestBug1490652ReadData(testOptions); DumpDictToFile(TestBug1490652DataRead, "Read.log"); //Verify return(TestBug1490652Verify(testOptions)); } catch (Exception e) { Logger.ErrorFormat("Dump topic got exception:{0}\r\ninput parameter: {1} \r\n" , e.FormatException(), testOptions.ToString()); return(false); } }
internal void Parse(string[] args) { if (args == null || args.Length <= 0) { throw new ArgumentException("Please provide verb."); } Verb = args[0].ToLowerInvariant(); if ("topic" == Verb.ToLowerInvariant()) ActiveSubOption = new TopicHelperArguments(); else if ("dumpdata" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeDataHelperArguments(); else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupHelperOptions(); else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant() || "consumegroupm" == Verb.ToLowerInvariant() || "consumem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ConsumeGroupMonitorHelperOptions(); else if ("produceroundrobin" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceSimpleHelperOption(); else if ("test" == Verb.ToLowerInvariant()) ActiveSubOption = new TestHelperOptions(); else if ("producewrapper" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProducePerfTestHelperOption(); else if ("producem" == Verb.ToLowerInvariant() || KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new ProduceMonitorHelperOptions(); else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant()) ActiveSubOption = new JavaEventServerPerfTestHelperOptions(); else { throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb)); } }