public void Producer_CustomPartitioner(string bootstrapServers)
{
LogToFile("start Producer_CustomPartitioner");
const int PARTITION_COUNT = 42;
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
};
for (int j = 0; j < 3; ++j)
{
using (var topic = new TemporaryTopic(bootstrapServers, PARTITION_COUNT))
{
Action <DeliveryReport <string, string> > dh = (DeliveryReport <string, string> dr) =>
{
Assert.StartsWith($"test key ", dr.Message.Key);
Assert.StartsWith($"test val ", dr.Message.Value);
var expectedPartition = int.Parse(dr.Message.Key.Split(" ").Last());
Assert.Equal(ErrorCode.NoError, dr.Error.Code);
Assert.Equal(PersistenceStatus.Persisted, dr.Status);
Assert.Equal(topic.Name, dr.Topic);
Assert.Equal(expectedPartition, (int)dr.Partition);
Assert.True(dr.Offset >= 0);
Assert.Equal(TimestampType.CreateTime, dr.Message.Timestamp.Type);
Assert.True(Math.Abs((DateTime.UtcNow - dr.Message.Timestamp.UtcDateTime).TotalMinutes) < 1.0);
};
ProducerBuilder <string, string> producerBuilder = null;
switch (j)
{
case 0:
// Topic level custom partitioner.
producerBuilder = new ProducerBuilder <string, string>(producerConfig);
producerBuilder.SetPartitioner(topic.Name, (string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) =>
{
Assert.Equal(topic.Name, topicName);
var keyString = System.Text.UTF8Encoding.UTF8.GetString(keyData.ToArray());
return(int.Parse(keyString.Split(" ").Last()) % partitionCount);
});
break;
case 1:
// Default custom partitioner
producerBuilder = new ProducerBuilder <string, string>(producerConfig);
producerBuilder.SetDefaultPartitioner((string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) =>
{
Assert.Equal(topic.Name, topicName);
var keyString = System.Text.UTF8Encoding.UTF8.GetString(keyData.ToArray());
return(int.Parse(keyString.Split(" ").Last()) % partitionCount);
});
break;
case 2:
// Default custom partitioner in case where default topic config is present due to topic level config in top-level config.
var producerConfig2 = new ProducerConfig
{
BootstrapServers = bootstrapServers,
MessageTimeoutMs = 10000
};
producerBuilder = new ProducerBuilder <string, string>(producerConfig2);
producerBuilder.SetDefaultPartitioner((string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) =>
{
Assert.Equal(topic.Name, topicName);
var keyString = System.Text.UTF8Encoding.UTF8.GetString(keyData.ToArray());
return(int.Parse(keyString.Split(" ").Last()) % partitionCount);
});
break;
default:
Assert.True(false);
break;
}
using (var producer = producerBuilder.Build())
{
for (int i = 0; i < PARTITION_COUNT; ++i)
{
producer.Produce(
topic.Name,
new Message <string, string> {
Key = $"test key {i}", Value = $"test val {i}"
}, dh);
}
producer.Flush(TimeSpan.FromSeconds(10));
}
}
}
// Null key
using (var topic = new TemporaryTopic(bootstrapServers, PARTITION_COUNT))
using (var producer = new ProducerBuilder <Null, string>(producerConfig)
.SetDefaultPartitioner((string topicName, int partitionCount, ReadOnlySpan <byte> keyData, bool keyIsNull) =>
{
Assert.True(keyIsNull);
return(0);
})
.Build())
{
producer.Produce(topic.Name, new Message <Null, string> {
Value = "test value"
});
producer.Flush(TimeSpan.FromSeconds(10));
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_CustomPartitioner");
}
private static long BenchmarkProducerImpl(
string bootstrapServers,
string topic,
int nMessages,
int nTests,
int nHeaders,
bool useDeliveryHandler,
bool usePartitioner = false)
{
// mirrors the librdkafka performance test example.
var config = new ProducerConfig
{
BootstrapServers = bootstrapServers,
QueueBufferingMaxMessages = 2000000,
MessageSendMaxRetries = 3,
RetryBackoffMs = 500,
LingerMs = 100,
DeliveryReportFields = "none"
};
DeliveryResult <Null, byte[]> firstDeliveryReport = null;
Headers headers = null;
if (nHeaders > 0)
{
headers = new Headers();
for (int i = 0; i < nHeaders; ++i)
{
headers.Add($"header-{i+1}", new byte[] { (byte)i, (byte)(i + 1), (byte)(i + 2), (byte)(i + 3) });
}
}
var producerBuilder = new ProducerBuilder <Null, byte[]>(config);
if (usePartitioner)
{
producerBuilder.SetPartitioner(topic, new BenchmarkPartioner(useAllPartitions: false));
}
using (var producer = producerBuilder.Build())
{
for (var j = 0; j < nTests; j += 1)
{
Console.WriteLine($"{producer.Name} producing on {topic} " + (useDeliveryHandler ? "[Action<Message>]" : "[Task]"));
byte cnt = 0;
var val = new byte[100].Select(a => ++ cnt).ToArray();
// this avoids including connection setup, topic creation time, etc.. in result.
firstDeliveryReport = producer.ProduceAsync(topic, new Message <Null, byte[]> {
Value = val, Headers = headers
}).Result;
var startTime = DateTime.Now.Ticks;
if (useDeliveryHandler)
{
var autoEvent = new AutoResetEvent(false);
var msgCount = nMessages;
Action <DeliveryReport <Null, byte[]> > deliveryHandler = (DeliveryReport <Null, byte[]> deliveryReport) =>
{
if (deliveryReport.Error.IsError)
{
// Not interested in benchmark results in the (unlikely) event there is an error.
Console.WriteLine($"A error occured producing a message: {deliveryReport.Error.Reason}");
Environment.Exit(1); // note: exceptions do not currently propagate to calling code from a deliveryHandler method.
}
if (--msgCount == 0)
{
autoEvent.Set();
}
};
for (int i = 0; i < nMessages; i += 1)
{
try
{
producer.Produce(topic, new Message <Null, byte[]> {
Value = val, Headers = headers
}, deliveryHandler);
}
catch (ProduceException <Null, byte[]> ex)
{
if (ex.Error.Code == ErrorCode.Local_QueueFull)
{
producer.Poll(TimeSpan.FromSeconds(1));
i -= 1;
}
else
{
throw;
}
}
}
while (true)
{
if (autoEvent.WaitOne(TimeSpan.FromSeconds(1)))
{
break;
}
Console.WriteLine(msgCount);
}
}
else
{
try
{
var tasks = new Task[nMessages];
for (int i = 0; i < nMessages; i += 1)
{
tasks[i] = producer.ProduceAsync(topic, new Message <Null, byte[]> {
Value = val, Headers = headers
});
if (tasks[i].IsFaulted)
{
if (((ProduceException <Null, byte[]>)tasks[i].Exception.InnerException).Error.Code == ErrorCode.Local_QueueFull)
{
producer.Poll(TimeSpan.FromSeconds(1));
i -= 1;
}
else
{
// unexpected, abort benchmark test.
throw tasks[i].Exception;
}
}
}
Task.WaitAll(tasks);
}
catch (AggregateException ex)
{
Console.WriteLine(ex.Message);
}
}
var duration = DateTime.Now.Ticks - startTime;
Console.WriteLine($"Produced {nMessages} messages in {duration/10000.0:F0}ms");
Console.WriteLine($"{nMessages / (duration/10000.0):F0}k msg/s");
}
producer.Flush(TimeSpan.FromSeconds(10));
}
return(firstDeliveryReport.Offset);
}