static void ReadData(List <Uri> brokersUriList, ArgReadWriteKafka options)
{
if (string.IsNullOrWhiteSpace(options.ReadTopic))
{
Logger.WarnFormat($"{nameof(options.ReadTopic)} is null or empty! If you want to write, please set {nameof(options.IsWrite)} true.");
return;
}
var beginTime = DateTime.Now;
var connectedTime = beginTime;
var endTime = beginTime;
var rowsRead = 0;
Action readTopicRows = () =>
{
using (var router = new BrokerRouter(new KafkaOptions(brokersUriList.ToArray())))
{
using (var consumer = new Consumer(new ConsumerOptions(options.ReadTopic, router)))
{
connectedTime = DateTime.Now;
foreach (var message in consumer.Consume())
{
rowsRead++;
if (!options.OnlyShowCount)
{
var text = Encoding.UTF8.GetString(message.Value);
if (options.ShowRowHeader)
{
Console.WriteLine("Row[{0}]: {1}", rowsRead, text);
}
else
{
Console.WriteLine(text);
}
}
if (options.Rows > 0 && rowsRead >= options.Rows)
{
break;
}
}
endTime = DateTime.Now;
}
}
};
if (options.ReadingTimeout > 0)
{
Task.Run(readTopicRows).Wait(options.ReadingTimeout);
}
else
{
Task.Run(readTopicRows).Wait();
}
Logger.InfoFormat($"Read {rowsRead} rows in topic {options.ReadTopic}, brokers = {options.BrokerList}, used time = {(endTime - beginTime).TotalSeconds} s = {endTime - beginTime}, connection cost = {connectedTime - beginTime}");
}
static void WriteTestData(List <Uri> brokersUriList, ArgReadWriteKafka options)
{
var baseId = options.BaseId;
var maxRows = Math.Min(3000, Math.Max(100, options.Rows));
var maxId = options.MaxId < baseId ? baseId + (int)(maxRows * 0.6) : options.MaxId;
var idList = new HashSet <long>();
for (var k = 0; k < maxId - baseId; k++)
{
idList.Add(GerenateUserId(baseId, maxId));
}
Logger.Debug($"baseId = {baseId}, maxId = {maxId}, idList.count = {idList.Count}, to write rows = {options.Rows}");
if (!string.IsNullOrWhiteSpace(options.TopicIdUser))
{
var tableIdUser = new List <Message>();
foreach (var id in idList)
{
tableIdUser.Add(new Message(new RowIdUser {
Id = id, User = GerenateUserName(id)
}.ToString()));
}
WriteTopic(brokersUriList, options.TopicIdUser, tableIdUser);
}
if (string.IsNullOrWhiteSpace(options.TopicIdCount))
{
return;
}
var beginTime = DateTime.Now;
var endTime = options.RunningSeconds == 0 ? DateTime.MaxValue : beginTime + TimeSpan.FromSeconds(options.RunningSeconds);
var rows = options.Rows == 0 ? long.MaxValue : options.Rows;
var oneMessage = new Message(new RowIdCountTime().ToString());
var size = oneMessage.Value.Length;
var oneBatch = 1024 * 1024 / size;
if (options.Interval >= 100)
{
oneBatch = 1;
}
var tableIdCount = new List <Message>();
for (var k = 0; k < rows && DateTime.Now < endTime; k++)
{
var time = DateTime.Now.Add(TimeSpan.FromMilliseconds(Math.Max(options.Interval, 1)));
tableIdCount.Add(new Message(new RowIdCountTime {
Id = idList.ElementAt(random.Next(0, idList.Count - 1)), Count = 1, Time = time
}.ToString()));
if (tableIdCount.Count == oneBatch)
{
WriteTopic(brokersUriList, options.TopicIdCount, tableIdCount, true, false);
rows += tableIdCount.Count;
tableIdCount.Clear();
}
if (options.Interval > 0)
{
Thread.Sleep(options.Interval);
}
}
WriteTopic(brokersUriList, options.TopicIdCount, tableIdCount, tableIdCount.Count > 0, true);
}
