Exemple #1
0
        private ISet <string> PublishMessages(string topic, int count, bool enableBatch)
        {
            ISet <string> keys    = new HashSet <string>();
            var           builder = new ProducerConfigBuilder <byte[]>()
                                    .Topic(topic)
                                    .MessageRoutingMode(Common.MessageRoutingMode.RoundRobinMode)
                                    .MaxPendingMessages(count)
                                    .BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(80000));

            if (enableBatch)
            {
                builder.EnableBatching(true);
                builder.BatchingMaxMessages(count);
            }
            else
            {
                builder.EnableBatching(false);
            }

            var producer = _client.NewProducer(builder);

            for (var i = 0; i < count; i++)
            {
                var key  = "key" + i;
                var data = Encoding.UTF8.GetBytes("my-message-" + i);
                producer.NewMessage().Key(key).Value(data).Send();
                keys.Add(key);
            }
            producer.Flush();
            return(keys);
        }
        private Producer <byte[]> CreateProducer(string topic, bool enableBatch, int batchSize = 500)
        {
            var pBuilder = new ProducerConfigBuilder <byte[]>();

            pBuilder.Topic(topic);
            if (enableBatch)
            {
                pBuilder.EnableBatching(true);
                pBuilder.BatchBuilder(IBatcherBuilder.KeyBased(_client.ActorSystem));
                pBuilder.BatchingMaxMessages(batchSize);
                pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(5000));
            }

            return(_client.NewProducer(pBuilder));
        }
Exemple #3
0
        public void ProduceAndConsumeBatch()
        {
            var r       = new Random(0);
            var byteKey = new byte[1000];

            r.NextBytes(byteKey);

            var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
                                  .Topic(_topic)
                                  .ForceTopicCreation(true)
                                  .SubscriptionName($"Batch-subscriber-{Guid.NewGuid()}");
            var consumer = _client.NewConsumer(consumerBuilder);


            var producerBuilder = new ProducerConfigBuilder <byte[]>();

            producerBuilder.Topic(_topic);
            producerBuilder.EnableBatching(true);
            producerBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(10000));
            producerBuilder.BatchingMaxMessages(5);
            var producer = _client.NewProducer(producerBuilder);

            for (var i = 0; i < 5; i++)
            {
                producer.NewMessage().KeyBytes(byteKey)
                .Properties(new Dictionary <string, string> {
                    { "KeyBytes", Encoding.UTF8.GetString(byteKey) }
                })
                .Value(Encoding.UTF8.GetBytes($"TestMessage-{i}"))
                .Send();
            }

            Thread.Sleep(TimeSpan.FromSeconds(5));
            for (var i = 0; i < 5; i++)
            {
                var message = consumer.Receive();
                Assert.Equal(byteKey, message.KeyBytes);
                Assert.True(message.HasBase64EncodedKey());
                var receivedMessage = Encoding.UTF8.GetString(message.Data);
                _output.WriteLine($"Received message: [{receivedMessage}]");
                Assert.Equal($"TestMessage-{i}", receivedMessage);
            }

            producer.Close();
            consumer.Close();
        }
Exemple #4
0
        private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout)
        {
            _output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}");
            var topic = "testNegativeAcks-" + DateTime.Now.Ticks;

            var builder = new ConsumerConfigBuilder <byte[]>()
                          .Topic(topic)
                          .SubscriptionName($"sub1-{Guid.NewGuid()}")
                          .AckTimeout(TimeSpan.FromMilliseconds(ackTimeout))
                          .ForceTopicCreation(true)
                          .AcknowledgmentGroupTime(0)
                          .NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis))
                          .SubscriptionType(subscriptionType);
            var consumer = _client.NewConsumer(builder);

            var pBuilder = new ProducerConfigBuilder <byte[]>();

            pBuilder.Topic(topic);
            if (batching)
            {
                pBuilder.EnableBatching(batching);
                pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis));
                pBuilder.BatchingMaxMessages(10);
            }
            var producer = _client.NewProducer(pBuilder);

            ISet <string> sentMessages = new HashSet <string>();

            const int n = 10;

            for (var i = 0; i < n; i++)
            {
                var value = "test-" + i;
                producer.Send(Encoding.UTF8.GetBytes(value));
                sentMessages.Add(value);
            }
            Thread.Sleep(TimeSpan.FromSeconds(10));
            for (var i = 0; i < n; i++)
            {
                var msg = consumer.Receive();
                if (msg != null)
                {
                    var ms = Encoding.UTF8.GetString(msg.Data);
                    consumer.NegativeAcknowledge(msg);
                    _output.WriteLine(ms);
                }
            }

            ISet <string> receivedMessages = new HashSet <string>();

            Thread.Sleep(TimeSpan.FromSeconds(10));
            // All the messages should be received again
            for (var i = 0; i < n; i++)
            {
                var msg = consumer.Receive();
                if (msg != null)
                {
                    var ms = Encoding.UTF8.GetString(msg.Data);
                    _output.WriteLine(ms);
                    receivedMessages.Add(ms);
                    consumer.Acknowledge(msg);
                }
            }

            Assert.Equal(sentMessages, receivedMessages);
            var nu = consumer.Receive();

            // There should be no more messages
            Assert.Null(nu);
            producer.Close();
            consumer.Close();
        }