private static void TestBug1490652ReadData(TestHelperOptions testOptions)
{
KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration()
{
FetchSize = KafkaSimpleManagerConfiguration.DefaultFetchSize,
BufferSize = KafkaSimpleManagerConfiguration.DefaultBufferSize,
MaxWaitTime = 0,
MinWaitBytes = 0,
Zookeeper = testOptions.Zookeeper
};
config.Verify();
using (KafkaSimpleManager <int, Message> kafkaSimpleManager = new KafkaSimpleManager <int, Message>(config))
{
TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(0, "ClientID", 0, testOptions.Topic, true);
PartitionCount = topicMetadata.PartitionsMetadata.Count();
for (int i = 0; i < PartitionCount; i++)
{
#region Get real offset and adjust
long earliest = 0;
long latest = 0;
long offsetBase = 0;
OffsetHelper.GetAdjustedOffset <int, Message>(testOptions.Topic, kafkaSimpleManager, i, KafkaOffsetType.Earliest, 0,
0, out earliest, out latest, out offsetBase);
#endregion
TestBug1490652DataRead.Add(i, ConsumeDataOfOnePartitionTotally <int, Message>(testOptions.Topic, kafkaSimpleManager, i, KafkaOffsetType.Earliest,
0, 0, latest, 0, 100, -1, "DumpLog.log"));
}
}
}
internal static void ConsumeDataSimple(ConsumeDataHelperArguments dumpdataOptions)
{
correlationID = 0;
totalCountUTF8 = 0;
totalCountOriginal = 0;
totalCount = 0;
lastNotifytotalCount = 0;
KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration()
{
FetchSize = dumpdataOptions.FetchSize,
BufferSize = dumpdataOptions.BufferSize,
MaxWaitTime = dumpdataOptions.MaxWaitTime,
MinWaitBytes = dumpdataOptions.MinWaitBytes,
Zookeeper = dumpdataOptions.Zookeeper
};
config.Verify();
bool finish = false;
try
{
using (KafkaSimpleManager <int, Message> kafkaSimpleManager = new KafkaSimpleManager <int, Message>(config))
{
TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(0, ClientID, correlationID++, dumpdataOptions.Topic, true);
while (true)
{
try
{
for (int i = 0; i <= topicMetadata.PartitionsMetadata.Max(r => r.PartitionId); i++)
{
if (dumpdataOptions.PartitionIndex == -1 || i == dumpdataOptions.PartitionIndex)
{
#region Get real offset and adjust
long earliest = 0;
long latest = 0;
long offsetBase = 0;
OffsetHelper.GetAdjustedOffset <int, Message>(dumpdataOptions.Topic
, kafkaSimpleManager, i
, KafkaNetLibraryExample.ConvertOffsetType(dumpdataOptions.Offset)
, KafkaNetLibraryExample.ConvertOffset(dumpdataOptions.Offset)
, dumpdataOptions.LastMessagesCount, out earliest, out latest, out offsetBase);
#endregion
Console.WriteLine("Topic:{0} Partition:{1} will read from {2} earliest:{3} latest:{4}", dumpdataOptions.Topic, i, offsetBase, earliest, latest);
finish = ConsumeDataOfOnePartition(kafkaSimpleManager, i, offsetBase, earliest, latest, dumpdataOptions);
if (finish)
{
break;
}
}
}
finish = true;
}
catch (Exception ex)
{
Logger.ErrorFormat("ConsumeDataSimple Got exception, will refresh metadata. {0}", ex.FormatException());
kafkaSimpleManager.RefreshMetadata(0, ClientID, correlationID++, dumpdataOptions.Topic, true);
}
if (finish)
{
break;
}
}
}
Logger.InfoFormat("Topic:{0} Finish Read. totalCount:{1} ", dumpdataOptions.Topic, totalCount);
}
catch (Exception ex)
{
Logger.ErrorFormat("ConsumeDataSimple Got exception:{0}\r\ninput parameter: {1}", ex.FormatException(), dumpdataOptions.ToString());
}
}
