public static void Main(string[] args)
{
var config = new ProducerConfig {
BootstrapServers = args[0]
};
using (var producer = new ProducerBuilder <string, string>(config).Build())
using (var producer2 = new DependentProducerBuilder <Null, int>(producer.Handle).Build())
{
// write (string, string) data to topic "first-topic".
producer.ProduceAsync("first-topic", new Message <string, string> {
Key = "my-key-value", Value = "my-value"
});
// write (null, int) data to topic "second-data" using the same underlying broker connections.
producer2.ProduceAsync("second-topic", new Message <Null, int> {
Value = 42
});
// producers are not tied to topics. Although it's unusual that you might want to
// do so, you can use different producers to write to the same topic.
producer2.ProduceAsync("first-topic", new Message <Null, int> {
Value = 107
});
// As the Tasks returned by ProduceAsync are not waited on there will still be messages in flight.
producer.Flush(TimeSpan.FromSeconds(10));
}
}
/// <summary>
/// Creates a producer
/// </summary>
public KafkaProducer(
Handle producerHandle,
object valueSerializer,
ILogger logger)
{
this.valueSerializer = valueSerializer;
this.logger = logger;
var builder = new DependentProducerBuilder <TKey, TValue>(producerHandle);
if (valueSerializer != null)
{
if (valueSerializer is IAsyncSerializer <TValue> asyncSerializer)
{
builder.SetValueSerializer(asyncSerializer);
}
else if (valueSerializer is ISerializer <TValue> syncSerializer)
{
builder.SetValueSerializer(syncSerializer);
}
else
{
throw new ArgumentException($"Value serializer must implement either IAsyncSerializer or ISerializer. Type {valueSerializer.GetType().Name} does not", nameof(valueSerializer));
}
}
this.producer = builder.Build();
}
public void Producer_ProduceAsync_HighConcurrency(string bootstrapServers)
{
LogToFile("start Producer_ProduceAsync_HighConcurrency");
ThreadPool.GetMaxThreads(out int originalWorkerThreads, out int originalCompletionPortThreads);
ThreadPool.GetMinThreads(out int workerThreads, out int completionPortThreads);
ThreadPool.SetMaxThreads(workerThreads, completionPortThreads);
ThreadPool.GetMaxThreads(out workerThreads, out completionPortThreads);
var pConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var tempTopic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <Null, string>(pConfig)
.SetValueSerializer(new SimpleAsyncSerializer().SyncOverAsync())
.Build())
using (var dProducer = new DependentProducerBuilder <Null, string>(producer.Handle)
.SetValueSerializer(new SimpleAsyncSerializer())
.Build())
{
var tasks = new List <Task>();
int N = workerThreads + 2;
for (int i = 0; i < N; ++i)
{
tasks.Add(producer.ProduceAsync(tempTopic.Name, new Message <Null, string> {
Value = "test"
}));
}
Task.WaitAll(tasks.ToArray());
for (int i = 0; i < N; ++i)
{
tasks.Add(dProducer.ProduceAsync(tempTopic.Name, new Message <Null, string> {
Value = "test"
}));
}
Task.WaitAll(tasks.ToArray());
}
ThreadPool.SetMaxThreads(originalWorkerThreads, originalCompletionPortThreads);
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_ProduceAsync_HighConcurrency");
}
public void Producer_Produce_Async(string bootstrapServers)
{
LogToFile("start Producer_Produce_Async");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var testTopic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <Null, string>(producerConfig)
.SetValueSerializer(new SimpleAsyncSerializer())
.Build())
using (var dProducer = new DependentProducerBuilder <string, Null>(producer.Handle)
.SetKeySerializer(new SimpleAsyncSerializer())
.Build())
{
Assert.Throws <ProduceException <Null, string> >(
() => producer.Produce(testTopic.Name, new Message <Null, string> {
Value = "test"
}));
Assert.Throws <ProduceException <Null, string> >(
() => producer.Produce(testTopic.Name, new Message <Null, string> {
Value = "test"
}, dr => { Assert.True(false); }));
Assert.Throws <ProduceException <string, Null> >(
() => dProducer.Produce(testTopic.Name, new Message <string, Null> {
Key = "test"
}));
Assert.Throws <ProduceException <string, Null> >(
() => dProducer.Produce(testTopic.Name, new Message <string, Null> {
Key = "test"
}, dr => { Assert.True(false); }));
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_Produce_Async");
}
public void Producer_Handles(string bootstrapServers)
{
LogToFile("start Producer_Handles");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer1 = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var producer2 = new DependentProducerBuilder <string, string>(producer1.Handle).Build())
using (var producer3 = new DependentProducerBuilder <byte[], byte[]>(producer1.Handle).Build())
using (var producer4 = new DependentProducerBuilder <int, string>(producer2.Handle).Build())
using (var producer5 = new DependentProducerBuilder <int, int>(producer3.Handle).Build())
using (var producer6 = new DependentProducerBuilder <string, byte[]>(producer4.Handle).Build())
using (var producer7 = new ProducerBuilder <double, double>(producerConfig).Build())
using (var adminClient = new DependentAdminClientBuilder(producer7.Handle).Build())
{
var r1 = producer1.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 42 }, Value = new byte[] { 33 }
}).Result;
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
var r2 = producer2.ProduceAsync(topic.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Result;
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
var r3 = producer3.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 40 }, Value = new byte[] { 31 }
}).Result;
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
var r4 = producer4.ProduceAsync(topic.Name, new Message <int, string> {
Key = 42, Value = "mellow world"
}).Result;
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
var r5 = producer5.ProduceAsync(topic.Name, new Message <int, int> {
Key = int.MaxValue, Value = int.MinValue
}).Result;
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
var r6 = producer6.ProduceAsync(topic.Name, new Message <string, byte[]> {
Key = "yellow mould", Value = new byte[] { 69 }
}).Result;
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
var r7 = producer7.ProduceAsync(topic.Name, new Message <double, double> {
Key = 44.0, Value = 234.4
}).Result;
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
var topicMetadata = adminClient.GetMetadata(singlePartitionTopic, TimeSpan.FromSeconds(10));
Assert.Single(topicMetadata.Topics);
// implicitly check this does not throw.
}
var consumerConfig = new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString()
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var r1 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
}
using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 1));
var r2 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
Assert.Equal(1, r2.Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 2));
var r3 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
Assert.Equal(2, r3.Offset);
}
using (var consumer = new ConsumerBuilder <int, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 3));
var r4 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
Assert.Equal(3, r4.Offset);
}
using (var consumer = new ConsumerBuilder <int, int>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 4));
var r5 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
Assert.Equal(4, r5.Offset);
}
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 5));
var r6 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
Assert.Equal(5, r6.Offset);
}
using (var consumer = new ConsumerBuilder <double, double>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 6));
var r7 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
Assert.Equal(6, r7.Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_Handles");
}
public ProducerBuilderWrapper(KafkaProducerConnection connection)
{
_dependentProducerBuilder = new DependentProducerBuilder <TKey, TValue>(connection.Handle);
}
