static void ReadData(List <Uri> brokersUriList, ArgReadWriteKafka options) { if (string.IsNullOrWhiteSpace(options.ReadTopic)) { Logger.WarnFormat($"{nameof(options.ReadTopic)} is null or empty! If you want to write, please set {nameof(options.IsWrite)} true."); return; } var beginTime = DateTime.Now; var connectedTime = beginTime; var endTime = beginTime; var rowsRead = 0; Action readTopicRows = () => { using (var router = new BrokerRouter(new KafkaOptions(brokersUriList.ToArray()))) { using (var consumer = new Consumer(new ConsumerOptions(options.ReadTopic, router))) { connectedTime = DateTime.Now; foreach (var message in consumer.Consume()) { rowsRead++; if (!options.OnlyShowCount) { var text = Encoding.UTF8.GetString(message.Value); if (options.ShowRowHeader) { Console.WriteLine("Row[{0}]: {1}", rowsRead, text); } else { Console.WriteLine(text); } } if (options.Rows > 0 && rowsRead >= options.Rows) { break; } } endTime = DateTime.Now; } } }; if (options.ReadingTimeout > 0) { Task.Run(readTopicRows).Wait(options.ReadingTimeout); } else { Task.Run(readTopicRows).Wait(); } Logger.InfoFormat($"Read {rowsRead} rows in topic {options.ReadTopic}, brokers = {options.BrokerList}, used time = {(endTime - beginTime).TotalSeconds} s = {endTime - beginTime}, connection cost = {connectedTime - beginTime}"); }
static void WriteTestData(List <Uri> brokersUriList, ArgReadWriteKafka options) { var baseId = options.BaseId; var maxRows = Math.Min(3000, Math.Max(100, options.Rows)); var maxId = options.MaxId < baseId ? baseId + (int)(maxRows * 0.6) : options.MaxId; var idList = new HashSet <long>(); for (var k = 0; k < maxId - baseId; k++) { idList.Add(GerenateUserId(baseId, maxId)); } Logger.Debug($"baseId = {baseId}, maxId = {maxId}, idList.count = {idList.Count}, to write rows = {options.Rows}"); if (!string.IsNullOrWhiteSpace(options.TopicIdUser)) { var tableIdUser = new List <Message>(); foreach (var id in idList) { tableIdUser.Add(new Message(new RowIdUser { Id = id, User = GerenateUserName(id) }.ToString())); } WriteTopic(brokersUriList, options.TopicIdUser, tableIdUser); } if (string.IsNullOrWhiteSpace(options.TopicIdCount)) { return; } var beginTime = DateTime.Now; var endTime = options.RunningSeconds == 0 ? DateTime.MaxValue : beginTime + TimeSpan.FromSeconds(options.RunningSeconds); var rows = options.Rows == 0 ? long.MaxValue : options.Rows; var oneMessage = new Message(new RowIdCountTime().ToString()); var size = oneMessage.Value.Length; var oneBatch = 1024 * 1024 / size; if (options.Interval >= 100) { oneBatch = 1; } var tableIdCount = new List <Message>(); for (var k = 0; k < rows && DateTime.Now < endTime; k++) { var time = DateTime.Now.Add(TimeSpan.FromMilliseconds(Math.Max(options.Interval, 1))); tableIdCount.Add(new Message(new RowIdCountTime { Id = idList.ElementAt(random.Next(0, idList.Count - 1)), Count = 1, Time = time }.ToString())); if (tableIdCount.Count == oneBatch) { WriteTopic(brokersUriList, options.TopicIdCount, tableIdCount, true, false); rows += tableIdCount.Count; tableIdCount.Clear(); } if (options.Interval > 0) { Thread.Sleep(options.Interval); } } WriteTopic(brokersUriList, options.TopicIdCount, tableIdCount, tableIdCount.Count > 0, true); }