public void Producer_CustomPartitioner(string bootstrapServers) { LogToFile("start Producer_CustomPartitioner"); const int PARTITION_COUNT = 42; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers, }; for (int j = 0; j < 3; ++j) { using (var topic = new TemporaryTopic(bootstrapServers, PARTITION_COUNT)) { Action <DeliveryReport <string, string> > dh = (DeliveryReport <string, string> dr) => { Assert.StartsWith($"test key ", dr.Message.Key); Assert.StartsWith($"test val ", dr.Message.Value); var expectedPartition = int.Parse(dr.Message.Key.Split(" ").Last()); Assert.Equal(ErrorCode.NoError, dr.Error.Code); Assert.Equal(PersistenceStatus.Persisted, dr.Status); Assert.Equal(topic.Name, dr.Topic); Assert.Equal(expectedPartition, (int)dr.Partition); Assert.True(dr.Offset >= 0); Assert.Equal(TimestampType.CreateTime, dr.Message.Timestamp.Type); Assert.True(Math.Abs((DateTime.UtcNow - dr.Message.Timestamp.UtcDateTime).TotalMinutes) < 1.0); }; ProducerBuilder <string, string> producerBuilder = null; switch (j) { case 0: // Topic level custom partitioner. producerBuilder = new ProducerBuilder <string, string>(producerConfig); producerBuilder.SetPartitioner(topic.Name, (string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) => { Assert.Equal(topic.Name, topicName); var keyString = System.Text.UTF8Encoding.UTF8.GetString(keyData.ToArray()); return(int.Parse(keyString.Split(" ").Last()) % partitionCount); }); break; case 1: // Default custom partitioner producerBuilder = new ProducerBuilder <string, string>(producerConfig); producerBuilder.SetDefaultPartitioner((string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) => { Assert.Equal(topic.Name, topicName); var keyString = System.Text.UTF8Encoding.UTF8.GetString(keyData.ToArray()); return(int.Parse(keyString.Split(" ").Last()) % partitionCount); }); break; case 2: // Default custom partitioner in case where default topic config is present due to topic level config in top-level config. var producerConfig2 = new ProducerConfig { BootstrapServers = bootstrapServers, MessageTimeoutMs = 10000 }; producerBuilder = new ProducerBuilder <string, string>(producerConfig2); producerBuilder.SetDefaultPartitioner((string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) => { Assert.Equal(topic.Name, topicName); var keyString = System.Text.UTF8Encoding.UTF8.GetString(keyData.ToArray()); return(int.Parse(keyString.Split(" ").Last()) % partitionCount); }); break; default: Assert.True(false); break; } using (var producer = producerBuilder.Build()) { for (int i = 0; i < PARTITION_COUNT; ++i) { producer.Produce( topic.Name, new Message <string, string> { Key = $"test key {i}", Value = $"test val {i}" }, dh); } producer.Flush(TimeSpan.FromSeconds(10)); } } } // Null key using (var topic = new TemporaryTopic(bootstrapServers, PARTITION_COUNT)) using (var producer = new ProducerBuilder <Null, string>(producerConfig) .SetDefaultPartitioner((string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) => { Assert.True(keyIsNull); return(0); }) .Build()) { producer.Produce(topic.Name, new Message <Null, string> { Value = "test value" }); producer.Flush(TimeSpan.FromSeconds(10)); } Assert.Equal(0, Library.HandleCount); LogToFile("end Producer_CustomPartitioner"); }
private static long BenchmarkProducerImpl( string bootstrapServers, string topic, int nMessages, int nTests, int nHeaders, bool useDeliveryHandler, bool usePartitioner = false) { // mirrors the librdkafka performance test example. var config = new ProducerConfig { BootstrapServers = bootstrapServers, QueueBufferingMaxMessages = 2000000, MessageSendMaxRetries = 3, RetryBackoffMs = 500, LingerMs = 100, DeliveryReportFields = "none" }; DeliveryResult <Null, byte[]> firstDeliveryReport = null; Headers headers = null; if (nHeaders > 0) { headers = new Headers(); for (int i = 0; i < nHeaders; ++i) { headers.Add($"header-{i+1}", new byte[] { (byte)i, (byte)(i + 1), (byte)(i + 2), (byte)(i + 3) }); } } var producerBuilder = new ProducerBuilder <Null, byte[]>(config); if (usePartitioner) { producerBuilder.SetPartitioner(topic, new BenchmarkPartioner(useAllPartitions: false)); } using (var producer = producerBuilder.Build()) { for (var j = 0; j < nTests; j += 1) { Console.WriteLine($"{producer.Name} producing on {topic} " + (useDeliveryHandler ? "[Action<Message>]" : "[Task]")); byte cnt = 0; var val = new byte[100].Select(a => ++ cnt).ToArray(); // this avoids including connection setup, topic creation time, etc.. in result. firstDeliveryReport = producer.ProduceAsync(topic, new Message <Null, byte[]> { Value = val, Headers = headers }).Result; var startTime = DateTime.Now.Ticks; if (useDeliveryHandler) { var autoEvent = new AutoResetEvent(false); var msgCount = nMessages; Action <DeliveryReport <Null, byte[]> > deliveryHandler = (DeliveryReport <Null, byte[]> deliveryReport) => { if (deliveryReport.Error.IsError) { // Not interested in benchmark results in the (unlikely) event there is an error. Console.WriteLine($"A error occured producing a message: {deliveryReport.Error.Reason}"); Environment.Exit(1); // note: exceptions do not currently propagate to calling code from a deliveryHandler method. } if (--msgCount == 0) { autoEvent.Set(); } }; for (int i = 0; i < nMessages; i += 1) { try { producer.Produce(topic, new Message <Null, byte[]> { Value = val, Headers = headers }, deliveryHandler); } catch (ProduceException <Null, byte[]> ex) { if (ex.Error.Code == ErrorCode.Local_QueueFull) { producer.Poll(TimeSpan.FromSeconds(1)); i -= 1; } else { throw; } } } while (true) { if (autoEvent.WaitOne(TimeSpan.FromSeconds(1))) { break; } Console.WriteLine(msgCount); } } else { try { var tasks = new Task[nMessages]; for (int i = 0; i < nMessages; i += 1) { tasks[i] = producer.ProduceAsync(topic, new Message <Null, byte[]> { Value = val, Headers = headers }); if (tasks[i].IsFaulted) { if (((ProduceException <Null, byte[]>)tasks[i].Exception.InnerException).Error.Code == ErrorCode.Local_QueueFull) { producer.Poll(TimeSpan.FromSeconds(1)); i -= 1; } else { // unexpected, abort benchmark test. throw tasks[i].Exception; } } } Task.WaitAll(tasks); } catch (AggregateException ex) { Console.WriteLine(ex.Message); } } var duration = DateTime.Now.Ticks - startTime; Console.WriteLine($"Produced {nMessages} messages in {duration/10000.0:F0}ms"); Console.WriteLine($"{nMessages / (duration/10000.0):F0}k msg/s"); } producer.Flush(TimeSpan.FromSeconds(10)); } return(firstDeliveryReport.Offset); }