private static bool TestBug1490652Verify(TestHelperOptions testOptions)
{
bool verifyResult = true;
foreach (KeyValuePair <int, Dictionary <int, string> > kv in TestBug1490652DataSent)
{
if (!TestBug1490652DataRead.ContainsKey(kv.Key))
{
verifyResult = false;
Logger.ErrorFormat("Result doesn't contains data for partition {0} ", kv.Key);
}
else
{
Dictionary <int, string> expectedResultOfOnePartition = kv.Value;
Dictionary <int, string> resultOfOnePartition = TestBug1490652DataRead[kv.Key];
foreach (KeyValuePair <int, string> kv2 in expectedResultOfOnePartition)
{
if (resultOfOnePartition[kv2.Key] != kv2.Value)
{
verifyResult = false;
Logger.ErrorFormat("Result doesn't match for partition {0} offset {1} data {2} vs {3} ", kv.Key, kv2.Key, kv2.Value, resultOfOnePartition[kv2.Key]);
}
}
}
}
return(verifyResult);
}
private static void TestBug1490652ReadData(TestHelperOptions testOptions)
{
KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration()
{
FetchSize = KafkaSimpleManagerConfiguration.DefaultFetchSize,
BufferSize = KafkaSimpleManagerConfiguration.DefaultBufferSize,
MaxWaitTime = 0,
MinWaitBytes = 0,
Zookeeper = testOptions.Zookeeper
};
config.Verify();
using (KafkaSimpleManager <int, Message> kafkaSimpleManager = new KafkaSimpleManager <int, Message>(config))
{
TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(0, "ClientID", 0, testOptions.Topic, true);
PartitionCount = topicMetadata.PartitionsMetadata.Count();
for (int i = 0; i < PartitionCount; i++)
{
#region Get real offset and adjust
long earliest = 0;
long latest = 0;
long offsetBase = 0;
OffsetHelper.GetAdjustedOffset <int, Message>(testOptions.Topic, kafkaSimpleManager, i, KafkaOffsetType.Earliest, 0,
0, out earliest, out latest, out offsetBase);
#endregion
TestBug1490652DataRead.Add(i, ConsumeDataOfOnePartitionTotally <int, Message>(testOptions.Topic, kafkaSimpleManager, i, KafkaOffsetType.Earliest,
0, 0, latest, 0, 100, -1, "DumpLog.log"));
}
}
}
private static void TestBug1490652SendData(TestHelperOptions testOptions)
{
int correlationID = 0;
Random rand = new Random();
StringBuilder sb = new StringBuilder();
try
{
KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration()
{
Zookeeper = testOptions.Zookeeper,
MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize
};
config.Verify();
using (KafkaSimpleManager <int, Kafka.Client.Messages.Message> kafkaSimpleManager = new KafkaSimpleManager <int, Kafka.Client.Messages.Message>(config))
{
TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(0, "ClientID", correlationID++, testOptions.Topic, true);
PartitionCount = topicMetadata.PartitionsMetadata.Count();
List <ProducerData <int, Message> > listOfDataNeedSendInOneBatch = new List <ProducerData <int, Message> >();
for (int i = 0; i < PartitionCount; i++)
{
TestBug1490652DataSent.Add(i, new Dictionary <int, string>());
for (int j = 0; j < TestBug1490652MessageCountPerPartition; j++)
{
string val = KafkaClientHelperUtils.GetRandomString(testOptions.MessageSize);
byte[] bVal = System.Text.Encoding.UTF8.GetBytes(val);
//Set the key to partitionID, so it can directly fall into that partition.
Message message = new Message(bVal, CompressionCodecs.DefaultCompressionCodec);
listOfDataNeedSendInOneBatch.Add(new ProducerData <int, Message>(testOptions.Topic, i, message));
TestBug1490652DataSent[i].Add(j, val);
}
}
ProducerConfiguration producerConfig = new ProducerConfiguration(new List <BrokerConfiguration>()
{
})
{
PartitionerClass = ProducerConfiguration.DefaultPartitioner,
RequiredAcks = 1,
BufferSize = config.BufferSize,
ZooKeeper = config.ZookeeperConfig,
MaxMessageSize = Math.Max(config.MaxMessageSize, Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, testOptions.MessageSize))
};
producerConfig.SyncProducerOfOneBroker = 1;
Producer <int, Kafka.Client.Messages.Message> producer = new Producer <int, Kafka.Client.Messages.Message>(producerConfig);
producer.Send(listOfDataNeedSendInOneBatch);
}
}
catch (Exception ex)
{
Logger.ErrorFormat("Produce data Got exception:{0}\r\ninput parameter: {1}\r\n"
, ex.FormatException(), testOptions.ToString());
}
}
/// <summary>
/// Run one test case
/// </summary>
/// <param name="testOptions"></param>
public static void Run(TestHelperOptions testOptions)
{
bool testResult = true;
if (testOptions.Case.ToLowerInvariant() == "bug1490652")
{
testResult &= TestBug1490652(testOptions);
}
if (testResult)
{
Console.WriteLine("PASS");
}
else
{
Logger.Error("Test cases FAILED!");
}
}
/// <summary>
/// Previously , the wrong exception hit when:
/// One broker is leader of multiple partition(>=2).
/// Send a group of message, and they need go to different partitions of the same broker.
/// Preparation before run the case:
/// Assume we have 3 broker, then create topic as
/// .\bin\kafka.cmd topiccmd --create --topic mvlogsA --partition 4 --replication-factor 1 --zookeeper localhost
/// Then run this case by send more than 5 messages.
/// </summary>
/// <param name="testOptions"></param>
/// <returns></returns>
private static bool TestBug1490652(TestHelperOptions testOptions)
{
try
{
//Send data
TestBug1490652SendData(testOptions);
DumpDictToFile(TestBug1490652DataSent, "Sent.log");
//Read data
TestBug1490652ReadData(testOptions);
DumpDictToFile(TestBug1490652DataRead, "Read.log");
//Verify
return(TestBug1490652Verify(testOptions));
}
catch (Exception e)
{
Logger.ErrorFormat("Dump topic got exception:{0}\r\ninput parameter: {1} \r\n"
, e.FormatException(), testOptions.ToString());
return(false);
}
}
internal void Parse(string[] args)
{
if (args == null || args.Length <= 0)
{
throw new ArgumentException("Please provide verb.");
}
Verb = args[0].ToLowerInvariant();
if ("topic" == Verb.ToLowerInvariant())
ActiveSubOption = new TopicHelperArguments();
else if ("dumpdata" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeDataHelperArguments();
else if ("dumpdataasconsumergroup" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroup.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupHelperOptions();
else if ("latestoffsetofconsumergroup" == Verb.ToLowerInvariant()
|| "consumegroupm" == Verb.ToLowerInvariant()
|| "consumem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ConsumeGroupMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ConsumeGroupMonitorHelperOptions();
else if ("produceroundrobin" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceSimple.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceSimpleHelperOption();
else if ("test" == Verb.ToLowerInvariant())
ActiveSubOption = new TestHelperOptions();
else if ("producewrapper" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProducePerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProducePerfTestHelperOption();
else if ("producem" == Verb.ToLowerInvariant()
|| KafkaNETExampleType.ProduceMonitor.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new ProduceMonitorHelperOptions();
else if (KafkaNETExampleType.EventServerPerfTest.ToString().ToLowerInvariant() == Verb.ToLowerInvariant())
ActiveSubOption = new JavaEventServerPerfTestHelperOptions();
else
{
throw new ArgumentException(string.Format("The command verb {0} is not recoganized.", Verb));
}
}