public static double Run(Dictionary <string, int> parameters) { MessageCount = 0; var kafkaSetting = new KafkaSetting() .SetBootstrapServers(new Uri(Program.KafkaUri)) .SetAcks(1) .SetRetries(0) .SetCompression(CompressionCodes.none) .Set("fetch.message.max.bytes", 2000) //.Set("auto.offset.reset", "latest") .SetClientId("client-id") .SetGroupId("test-group"); foreach (var parameter in parameters) { kafkaSetting.Set(parameter.Key, parameter.Value); } var consumers = Enumerable.Range(1, 1) .Select(x => new KafkaConsumer <byte[]>(kafkaSetting, Program.Topic, new SimpleDesiralizer(), new MessageObserver())) .ToArray(); var counter = 0; const int stepMilliseconds = 1000; double avgRps = 0; var stopwatch = new Stopwatch(); stopwatch.Start(); while (true) { var prevCount = MessageCount; Thread.Sleep(TimeSpan.FromMilliseconds(stepMilliseconds)); var newCount = MessageCount; var rps = (double)(newCount - prevCount) / stepMilliseconds * 1000; if (avgRps > 0 || rps > 0) { counter++; avgRps = (double)newCount / counter / stepMilliseconds * 1000; } //Console.WriteLine(DiffTimestampManager.GetReport()); Program.Log($"MessageCount={newCount}, perSecond={rps}, avg={avgRps}"); if (Math.Abs(rps) < 1 && newCount > 0 || stopwatch.ElapsedMilliseconds > 60000) // { break; } } foreach (var consumer in consumers) { consumer.Dispose(); } return(avgRps); }
public static double Run(Dictionary <string, int> parameters) { requestCount = 0; successCount = 0; errorCount = 0; var kafkaSetting = new KafkaSetting() .SetBootstrapServers(new Uri(Program.KafkaUri)) .SetAcks(1) .SetRetries(0) //.Set("auto.commit.interval.ms", 28120) //.Set("session.timeout.ms", 41904) //.Set("message.max.bytes", 8416000) //.Set("message.copy.max.bytes", 920000) //.Set("receive.message.max.bytes", 92000000) //.Set("max.in.flight.requests.per.connection", 128000) //.Set("queue.buffering.max.messages", 9001000) //.Set("queue.buffering.max.kbytes", 1887535) //.Set("queue.buffering.max.ms", 20) //.Set("batch.num.messages", 500500) .SetClientId("client-id") .SetGroupId("test-group"); foreach (var parameter in parameters.Where(x => !x.Key.StartsWith("_"))) { kafkaSetting.Set(parameter.Key, parameter.Value); } double avgRps; try { using (var kafkaProducer = new KafkaProducer(kafkaSetting, OnMessageDelivered)) { var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:8888") }; var cancellationTokenSource = new CancellationTokenSource(); var cancellationToken = cancellationTokenSource.Token; var tasks = new List <Task>(); avgRps = 0; var watcherTask = new Task(() => { var counter = 0; while (!cancellationToken.IsCancellationRequested) { var prevSuccess = successCount; Thread.Sleep(stepMilliseconds); var newSuccess = successCount; var rps = (double)(newSuccess - prevSuccess) / stepMilliseconds * 1000; if (avgRps > 0 || rps > 0) { counter++; avgRps = (double)successCount / counter / stepMilliseconds * 1000; } Program.Log($"tasks= {tasks.Count}, success = {successCount}, error = {errorCount}, perSecond={rps}, avg={avgRps}"); } }, cancellationToken, TaskCreationOptions.LongRunning); watcherTask.Start(); for (var i = 0; i < 1; i++) { for (var j = 0; j < parameters["_tasks"]; j++) { var task = new Task(() => { SendingLoop(kafkaProducer, httpClient, cancellationToken); }, cancellationToken); task.Start(); tasks.Add(task); } } Thread.Sleep(60000); cancellationTokenSource.Cancel(); Task.WaitAll(tasks.ToArray()); } } catch (Exception e) { Console.WriteLine(e); return(0); } Program.Log($"success = {successCount}, all = {requestCount}"); return(avgRps); }