private ISet <string> PublishMessages(string topic, int count, bool enableBatch)
{
ISet <string> keys = new HashSet <string>();
var builder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.MessageRoutingMode(Common.MessageRoutingMode.RoundRobinMode)
.MaxPendingMessages(count)
.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(80000));
if (enableBatch)
{
builder.EnableBatching(true);
builder.BatchingMaxMessages(count);
}
else
{
builder.EnableBatching(false);
}
var producer = _client.NewProducer(builder);
for (var i = 0; i < count; i++)
{
var key = "key" + i;
var data = Encoding.UTF8.GetBytes("my-message-" + i);
producer.NewMessage().Key(key).Value(data).Send();
keys.Add(key);
}
producer.Flush();
return(keys);
}
private Producer <byte[]> CreateProducer(string topic, bool enableBatch, int batchSize = 500)
{
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
if (enableBatch)
{
pBuilder.EnableBatching(true);
pBuilder.BatchBuilder(IBatcherBuilder.KeyBased(_client.ActorSystem));
pBuilder.BatchingMaxMessages(batchSize);
pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(5000));
}
return(_client.NewProducer(pBuilder));
}
public void ProduceAndConsumeBatch()
{
var r = new Random(0);
var byteKey = new byte[1000];
r.NextBytes(byteKey);
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(_topic)
.ForceTopicCreation(true)
.SubscriptionName($"Batch-subscriber-{Guid.NewGuid()}");
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>();
producerBuilder.Topic(_topic);
producerBuilder.EnableBatching(true);
producerBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(10000));
producerBuilder.BatchingMaxMessages(5);
var producer = _client.NewProducer(producerBuilder);
for (var i = 0; i < 5; i++)
{
producer.NewMessage().KeyBytes(byteKey)
.Properties(new Dictionary <string, string> {
{ "KeyBytes", Encoding.UTF8.GetString(byteKey) }
})
.Value(Encoding.UTF8.GetBytes($"TestMessage-{i}"))
.Send();
}
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < 5; i++)
{
var message = consumer.Receive();
Assert.Equal(byteKey, message.KeyBytes);
Assert.True(message.HasBase64EncodedKey());
var receivedMessage = Encoding.UTF8.GetString(message.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.Equal($"TestMessage-{i}", receivedMessage);
}
producer.Close();
consumer.Close();
}
private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout)
{
_output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}");
var topic = "testNegativeAcks-" + DateTime.Now.Ticks;
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName($"sub1-{Guid.NewGuid()}")
.AckTimeout(TimeSpan.FromMilliseconds(ackTimeout))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0)
.NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis))
.SubscriptionType(subscriptionType);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
if (batching)
{
pBuilder.EnableBatching(batching);
pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis));
pBuilder.BatchingMaxMessages(10);
}
var producer = _client.NewProducer(pBuilder);
ISet <string> sentMessages = new HashSet <string>();
const int n = 10;
for (var i = 0; i < n; i++)
{
var value = "test-" + i;
producer.Send(Encoding.UTF8.GetBytes(value));
sentMessages.Add(value);
}
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
consumer.NegativeAcknowledge(msg);
_output.WriteLine(ms);
}
}
ISet <string> receivedMessages = new HashSet <string>();
Thread.Sleep(TimeSpan.FromSeconds(10));
// All the messages should be received again
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
_output.WriteLine(ms);
receivedMessages.Add(ms);
consumer.Acknowledge(msg);
}
}
Assert.Equal(sentMessages, receivedMessages);
var nu = consumer.Receive();
// There should be no more messages
Assert.Null(nu);
producer.Close();
consumer.Close();
}