private Producer <byte[]> CreateProducer(string topic, bool enableBatch, int batchSize = 500) { var pBuilder = new ProducerConfigBuilder <byte[]>(); pBuilder.Topic(topic); if (enableBatch) { pBuilder.EnableBatching(true); pBuilder.BatchBuilder(IBatcherBuilder.KeyBased(_client.ActorSystem)); pBuilder.BatchingMaxMessages(batchSize); pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(5000)); } return(_client.NewProducer(pBuilder)); }
public void ProduceAndConsumeBatch() { var r = new Random(0); var byteKey = new byte[1000]; r.NextBytes(byteKey); var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(_topic) .ForceTopicCreation(true) .SubscriptionName($"Batch-subscriber-{Guid.NewGuid()}"); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>(); producerBuilder.Topic(_topic); producerBuilder.EnableBatching(true); producerBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(10000)); producerBuilder.BatchingMaxMessages(5); var producer = _client.NewProducer(producerBuilder); for (var i = 0; i < 5; i++) { producer.NewMessage().KeyBytes(byteKey) .Properties(new Dictionary <string, string> { { "KeyBytes", Encoding.UTF8.GetString(byteKey) } }) .Value(Encoding.UTF8.GetBytes($"TestMessage-{i}")) .Send(); } Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < 5; i++) { var message = consumer.Receive(); Assert.Equal(byteKey, message.KeyBytes); Assert.True(message.HasBase64EncodedKey()); var receivedMessage = Encoding.UTF8.GetString(message.Data); _output.WriteLine($"Received message: [{receivedMessage}]"); Assert.Equal($"TestMessage-{i}", receivedMessage); } producer.Close(); consumer.Close(); }
private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout) { _output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}"); var topic = "testNegativeAcks-" + DateTime.Now.Ticks; var builder = new ConsumerConfigBuilder <byte[]>() .Topic(topic) .SubscriptionName($"sub1-{Guid.NewGuid()}") .AckTimeout(TimeSpan.FromMilliseconds(ackTimeout)) .ForceTopicCreation(true) .AcknowledgmentGroupTime(0) .NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis)) .SubscriptionType(subscriptionType); var consumer = _client.NewConsumer(builder); var pBuilder = new ProducerConfigBuilder <byte[]>(); pBuilder.Topic(topic); if (batching) { pBuilder.EnableBatching(batching); pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis)); pBuilder.BatchingMaxMessages(10); } var producer = _client.NewProducer(pBuilder); ISet <string> sentMessages = new HashSet <string>(); const int n = 10; for (var i = 0; i < n; i++) { var value = "test-" + i; producer.Send(Encoding.UTF8.GetBytes(value)); sentMessages.Add(value); } Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < n; i++) { var msg = consumer.Receive(); if (msg != null) { var ms = Encoding.UTF8.GetString(msg.Data); consumer.NegativeAcknowledge(msg); _output.WriteLine(ms); } } ISet <string> receivedMessages = new HashSet <string>(); Thread.Sleep(TimeSpan.FromSeconds(10)); // All the messages should be received again for (var i = 0; i < n; i++) { var msg = consumer.Receive(); if (msg != null) { var ms = Encoding.UTF8.GetString(msg.Data); _output.WriteLine(ms); receivedMessages.Add(ms); consumer.Acknowledge(msg); } } Assert.Equal(sentMessages, receivedMessages); var nu = consumer.Receive(); // There should be no more messages Assert.Null(nu); producer.Close(); consumer.Close(); }