public static void Main(string[] args) { var config = new ProducerConfig { BootstrapServers = args[0] }; using (var producer = new ProducerBuilder <string, string>(config).Build()) using (var producer2 = new DependentProducerBuilder <Null, int>(producer.Handle).Build()) { // write (string, string) data to topic "first-topic". producer.ProduceAsync("first-topic", new Message <string, string> { Key = "my-key-value", Value = "my-value" }); // write (null, int) data to topic "second-data" using the same underlying broker connections. producer2.ProduceAsync("second-topic", new Message <Null, int> { Value = 42 }); // producers are not tied to topics. Although it's unusual that you might want to // do so, you can use different producers to write to the same topic. producer2.ProduceAsync("first-topic", new Message <Null, int> { Value = 107 }); // As the Tasks returned by ProduceAsync are not waited on there will still be messages in flight. producer.Flush(TimeSpan.FromSeconds(10)); } }
/// <summary> /// Creates a producer /// </summary> public KafkaProducer( Handle producerHandle, object valueSerializer, ILogger logger) { this.valueSerializer = valueSerializer; this.logger = logger; var builder = new DependentProducerBuilder <TKey, TValue>(producerHandle); if (valueSerializer != null) { if (valueSerializer is IAsyncSerializer <TValue> asyncSerializer) { builder.SetValueSerializer(asyncSerializer); } else if (valueSerializer is ISerializer <TValue> syncSerializer) { builder.SetValueSerializer(syncSerializer); } else { throw new ArgumentException($"Value serializer must implement either IAsyncSerializer or ISerializer. Type {valueSerializer.GetType().Name} does not", nameof(valueSerializer)); } } this.producer = builder.Build(); }
public void Producer_ProduceAsync_HighConcurrency(string bootstrapServers) { LogToFile("start Producer_ProduceAsync_HighConcurrency"); ThreadPool.GetMaxThreads(out int originalWorkerThreads, out int originalCompletionPortThreads); ThreadPool.GetMinThreads(out int workerThreads, out int completionPortThreads); ThreadPool.SetMaxThreads(workerThreads, completionPortThreads); ThreadPool.GetMaxThreads(out workerThreads, out completionPortThreads); var pConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; using (var tempTopic = new TemporaryTopic(bootstrapServers, 1)) using (var producer = new ProducerBuilder <Null, string>(pConfig) .SetValueSerializer(new SimpleAsyncSerializer().SyncOverAsync()) .Build()) using (var dProducer = new DependentProducerBuilder <Null, string>(producer.Handle) .SetValueSerializer(new SimpleAsyncSerializer()) .Build()) { var tasks = new List <Task>(); int N = workerThreads + 2; for (int i = 0; i < N; ++i) { tasks.Add(producer.ProduceAsync(tempTopic.Name, new Message <Null, string> { Value = "test" })); } Task.WaitAll(tasks.ToArray()); for (int i = 0; i < N; ++i) { tasks.Add(dProducer.ProduceAsync(tempTopic.Name, new Message <Null, string> { Value = "test" })); } Task.WaitAll(tasks.ToArray()); } ThreadPool.SetMaxThreads(originalWorkerThreads, originalCompletionPortThreads); Assert.Equal(0, Library.HandleCount); LogToFile("end Producer_ProduceAsync_HighConcurrency"); }
public void Producer_Produce_Async(string bootstrapServers) { LogToFile("start Producer_Produce_Async"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; using (var testTopic = new TemporaryTopic(bootstrapServers, 1)) using (var producer = new ProducerBuilder <Null, string>(producerConfig) .SetValueSerializer(new SimpleAsyncSerializer()) .Build()) using (var dProducer = new DependentProducerBuilder <string, Null>(producer.Handle) .SetKeySerializer(new SimpleAsyncSerializer()) .Build()) { Assert.Throws <ProduceException <Null, string> >( () => producer.Produce(testTopic.Name, new Message <Null, string> { Value = "test" })); Assert.Throws <ProduceException <Null, string> >( () => producer.Produce(testTopic.Name, new Message <Null, string> { Value = "test" }, dr => { Assert.True(false); })); Assert.Throws <ProduceException <string, Null> >( () => dProducer.Produce(testTopic.Name, new Message <string, Null> { Key = "test" })); Assert.Throws <ProduceException <string, Null> >( () => dProducer.Produce(testTopic.Name, new Message <string, Null> { Key = "test" }, dr => { Assert.True(false); })); } Assert.Equal(0, Library.HandleCount); LogToFile("end Producer_Produce_Async"); }
public void Producer_Handles(string bootstrapServers) { LogToFile("start Producer_Handles"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) { using (var producer1 = new ProducerBuilder <byte[], byte[]>(producerConfig).Build()) using (var producer2 = new DependentProducerBuilder <string, string>(producer1.Handle).Build()) using (var producer3 = new DependentProducerBuilder <byte[], byte[]>(producer1.Handle).Build()) using (var producer4 = new DependentProducerBuilder <int, string>(producer2.Handle).Build()) using (var producer5 = new DependentProducerBuilder <int, int>(producer3.Handle).Build()) using (var producer6 = new DependentProducerBuilder <string, byte[]>(producer4.Handle).Build()) using (var producer7 = new ProducerBuilder <double, double>(producerConfig).Build()) using (var adminClient = new DependentAdminClientBuilder(producer7.Handle).Build()) { var r1 = producer1.ProduceAsync(topic.Name, new Message <byte[], byte[]> { Key = new byte[] { 42 }, Value = new byte[] { 33 } }).Result; Assert.Equal(new byte[] { 42 }, r1.Key); Assert.Equal(new byte[] { 33 }, r1.Value); Assert.Equal(0, r1.Offset); var r2 = producer2.ProduceAsync(topic.Name, new Message <string, string> { Key = "hello", Value = "world" }).Result; Assert.Equal("hello", r2.Key); Assert.Equal("world", r2.Value); var r3 = producer3.ProduceAsync(topic.Name, new Message <byte[], byte[]> { Key = new byte[] { 40 }, Value = new byte[] { 31 } }).Result; Assert.Equal(new byte[] { 40 }, r3.Key); Assert.Equal(new byte[] { 31 }, r3.Value); var r4 = producer4.ProduceAsync(topic.Name, new Message <int, string> { Key = 42, Value = "mellow world" }).Result; Assert.Equal(42, r4.Key); Assert.Equal("mellow world", r4.Value); var r5 = producer5.ProduceAsync(topic.Name, new Message <int, int> { Key = int.MaxValue, Value = int.MinValue }).Result; Assert.Equal(int.MaxValue, r5.Key); Assert.Equal(int.MinValue, r5.Value); var r6 = producer6.ProduceAsync(topic.Name, new Message <string, byte[]> { Key = "yellow mould", Value = new byte[] { 69 } }).Result; Assert.Equal("yellow mould", r6.Key); Assert.Equal(new byte[] { 69 }, r6.Value); var r7 = producer7.ProduceAsync(topic.Name, new Message <double, double> { Key = 44.0, Value = 234.4 }).Result; Assert.Equal(44.0, r7.Key); Assert.Equal(234.4, r7.Value); var topicMetadata = adminClient.GetMetadata(singlePartitionTopic, TimeSpan.FromSeconds(10)); Assert.Single(topicMetadata.Topics); // implicitly check this does not throw. } var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString() }; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0)); var r1 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(new byte[] { 42 }, r1.Key); Assert.Equal(new byte[] { 33 }, r1.Value); Assert.Equal(0, r1.Offset); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 1)); var r2 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal("hello", r2.Key); Assert.Equal("world", r2.Value); Assert.Equal(1, r2.Offset); } using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 2)); var r3 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(new byte[] { 40 }, r3.Key); Assert.Equal(new byte[] { 31 }, r3.Value); Assert.Equal(2, r3.Offset); } using (var consumer = new ConsumerBuilder <int, string>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 3)); var r4 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(42, r4.Key); Assert.Equal("mellow world", r4.Value); Assert.Equal(3, r4.Offset); } using (var consumer = new ConsumerBuilder <int, int>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 4)); var r5 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(int.MaxValue, r5.Key); Assert.Equal(int.MinValue, r5.Value); Assert.Equal(4, r5.Offset); } using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 5)); var r6 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal("yellow mould", r6.Key); Assert.Equal(new byte[] { 69 }, r6.Value); Assert.Equal(5, r6.Offset); } using (var consumer = new ConsumerBuilder <double, double>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 6)); var r7 = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(44.0, r7.Key); Assert.Equal(234.4, r7.Value); Assert.Equal(6, r7.Offset); } } Assert.Equal(0, Library.HandleCount); LogToFile("end Producer_Handles"); }
public ProducerBuilderWrapper(KafkaProducerConnection connection) { _dependentProducerBuilder = new DependentProducerBuilder <TKey, TValue>(connection.Handle); }