public void TestImmediateAckingTracker()
{
var builder = new ConsumerConfigBuilder <byte[]>();
builder.AcknowledgmentGroupTime(0);
builder.Topic($"TestAckTracker-{Guid.NewGuid()}");
builder.SubscriptionName($"TestAckTracker-sub-{Guid.NewGuid()}");
var conf = builder.ConsumerConfigurationData;
var consumer = _client.NewConsumer(builder);
var unack = _system.ActorOf(UnAckedChunckedMessageIdSequenceMap.Prop());
var tracker = _client.ActorSystem.ActorOf(PersistentAcknowledgmentsGroupingTracker <byte[]> .Prop(unack, consumer.ConsumerActor, consumer.ConsumerActor /*dummy*/, 1, consumer.ConsumerActor, conf));
var msg1 = new MessageId(5, 1, 0);
var msg2 = new MessageId(5, 2, 0);
var isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg1, CommandAck.AckType.Individual, new Dictionary <string, long>()));
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(FlushPending.Instance);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg2, CommandAck.AckType.Individual, new Dictionary <string, long>()));
// Since we were connected, the ack went out immediately
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.GracefulStop(TimeSpan.FromSeconds(1));
}
public void ProduceAndConsume()
{
var record1 = AvroSchema <SimpleRecord> .Of(typeof(SimpleRecord));
var consumerBuilder = new ConsumerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema1")
.SubscriptionName("test-sub");
var consumer = _client.NewConsumer(record1, consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema1");
var producer = _client.NewProducer(record1, producerBuilder);
producer.NewMessage()
.Value(new SimpleRecord {
Name = "Ebere Abanonu", Age = int.MaxValue
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
consumer.Unsubscribe();
var record2 = AvroSchema <SimpleRecord2> .Of(typeof(SimpleRecord2));
var consumerBuilder2 = new ConsumerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema2")
.SubscriptionName("test-sub");
var consumer1 = _client.NewConsumer(record2, consumerBuilder2);
var producerBuilder2 = new ProducerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema2");
var producer2 = _client.NewProducer(record2, producerBuilder2);
producer2.NewMessage()
.Value(new SimpleRecord2 {
Name = "Ebere", Age = int.MaxValue, Surname = "Abanonu"
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var msg = consumer1.Receive();
Assert.NotNull(msg);
consumer1.Acknowledge(msg);
consumer1.Unsubscribe();
producer.Close();
producer2.Close();
consumer.Close();
consumer1.Close();
}
public void ProduceCommitBatchedTest()
{
var txn = Txn;
var topic = $"{_topicOutput}-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.ForceTopicCreation(true)
.SubscriptionName($"test2")
.EnableBatchIndexAcknowledgment(true)
.SubscriptionInitialPosition(SubscriptionInitialPosition.Earliest);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
//.EnableBatching(true)
.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var txnMessageCnt = 0;
var messageCnt = 40;
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage(txn).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
txnMessageCnt++;
}
// Can't receive transaction messages before commit.
var message = consumer.Receive();
Assert.Null(message);
txn.Commit();
// txn1 messages could be received after txn1 committed
var receiveCnt = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < txnMessageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
receiveCnt++;
_output.WriteLine($"message receive count: {receiveCnt}");
}
Assert.Equal(txnMessageCnt, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
_output.WriteLine($"message commit test enableBatch {true}");
}
public virtual void ConsumerInstantiation()
{
var consumer = new ConsumerConfigBuilder <string>();
consumer.Topic("ConsumerInstantiation");
consumer.SubscriptionName($"test-sub-{Guid.NewGuid()}");
var stringConsumerBuilder = _client.NewConsumer(new StringSchema(), consumer);
Assert.NotNull(stringConsumerBuilder);
stringConsumerBuilder.Close();
}
public void TestUnAckMessageRedeliveryWithReceive()
{
var topic = $"persistent://public/default/async-unack-redelivery-{Guid.NewGuid()}";
var builder = new ConsumerConfigBuilder <byte[]>();
builder.Topic(topic);
builder.SubscriptionName("sub-TestUnAckMessageRedeliveryWithReceive");
builder.AckTimeout(TimeSpan.FromMilliseconds(8000));
builder.ForceTopicCreation(true);
builder.AcknowledgmentGroupTime(0);
builder.SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
var producer = _client.NewProducer(pBuilder);
const int messageCount = 10;
for (var i = 0; i < messageCount; i++)
{
var receipt = producer.Send(Encoding.UTF8.GetBytes("my-message-" + i));
_output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions {
WriteIndented = true
}));
}
var messageReceived = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; ++i)
{
var m = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(m.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.NotNull(receivedMessage);
messageReceived++;
}
Assert.Equal(10, messageReceived);
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < messageCount; i++)
{
var m = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(m.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.NotNull(receivedMessage);
messageReceived++;
}
Assert.Equal(20, messageReceived);
producer.Close();
consumer.Close();
}
public void TestLargeMessage()
{
//this.conf.MaxMessageSize = 5;
const int totalMessages = 3;
var topicName = $"persistent://public/default/my-topic1-{DateTimeHelper.CurrentUnixTimeMillis()}";
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(topicName)
.SubscriptionName("my-subscriber-name")
.AckTimeout(TimeSpan.FromMilliseconds(20000))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topicName)
.EnableChunking(true)
.MaxMessageSize(5);
var producer = _client.NewProducer(pBuilder);
IList <string> publishedMessages = new List <string>();
for (var i = 1; i < totalMessages; i++)
{
var message = CreateMessagePayload(i * 10);
publishedMessages.Add(message);
producer.Send(Encoding.UTF8.GetBytes(message));
}
IMessage <byte[]> msg = null;
ISet <string> messageSet = new HashSet <string>();
IList <IMessage <byte[]> > msgIds = new List <IMessage <byte[]> >();
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < totalMessages - 1; i++)
{
msg = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(msg.Data);
_output.WriteLine($"[{i}] - Published [{publishedMessages[i]}] Received message: [{receivedMessage}]");
var expectedMessage = publishedMessages[i];
TestMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
msgIds.Add(msg);
}
foreach (var msgId in msgIds)
{
consumer.Acknowledge(msgId);
}
producer.Close();
consumer.Close();
}
private Consumer <byte[]> CreateConsumer(string topic, string consumerSub, KeySharedPolicy keySharedPolicy = null)
{
var builder = new ConsumerConfigBuilder <byte[]>();
builder.Topic(topic);
builder.SubscriptionName(consumerSub);
builder.AckTimeout(TimeSpan.FromSeconds(10));
builder.ForceTopicCreation(true);
if (keySharedPolicy != null)
{
builder.KeySharedPolicy(keySharedPolicy);
}
builder.SubscriptionType(CommandSubscribe.SubType.KeyShared);
return(_client.NewConsumer(builder));
}
public void ProduceAndConsumeBatch()
{
var r = new Random(0);
var byteKey = new byte[1000];
r.NextBytes(byteKey);
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(_topic)
.ForceTopicCreation(true)
.SubscriptionName($"Batch-subscriber-{Guid.NewGuid()}");
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>();
producerBuilder.Topic(_topic);
producerBuilder.EnableBatching(true);
producerBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(10000));
producerBuilder.BatchingMaxMessages(5);
var producer = _client.NewProducer(producerBuilder);
for (var i = 0; i < 5; i++)
{
producer.NewMessage().KeyBytes(byteKey)
.Properties(new Dictionary <string, string> {
{ "KeyBytes", Encoding.UTF8.GetString(byteKey) }
})
.Value(Encoding.UTF8.GetBytes($"TestMessage-{i}"))
.Send();
}
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < 5; i++)
{
var message = consumer.Receive();
Assert.Equal(byteKey, message.KeyBytes);
Assert.True(message.HasBase64EncodedKey());
var receivedMessage = Encoding.UTF8.GetString(message.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.Equal($"TestMessage-{i}", receivedMessage);
}
producer.Close();
consumer.Close();
}
public void ZeroQueueSizeNormalConsumer()
{
string key = "nonZeroQueueSizeNormalConsumer";
// 1. Config
string topicName = "topic-" + key;
string subscriptionName = "my-ex-subscription-" + key;
string messagePredicate = "my-message-" + key + "-";
// 2. Create Producer
var pBuilder = new ProducerConfigBuilder <sbyte[]>()
.Topic(topicName)
.EnableBatching(false);
var producer = _client.NewProducer(pBuilder);
// 3. Create Consumer
var config = new ConsumerConfigBuilder <sbyte[]>()
.Topic(topicName)
.SubscriptionName(subscriptionName)
.ReceiverQueueSize(0);
var consumer = _client.NewConsumer(config);
// 3. producer publish messages
for (int i = 0; i < _totalMessages; i++)
{
var msg = messagePredicate + i;
_output.WriteLine("Producer produced: " + msg);
producer.Send(Encoding.UTF8.GetBytes(msg).ToSBytes());
}
// 4. Receiver receives the message
IMessage <sbyte[]> message;
for (int i = 0; i < _totalMessages; i++)
{
Assert.Equal(0, consumer.NumMessagesInQueue());
message = consumer.Receive();
var r = Encoding.UTF8.GetString(message.Data.ToBytes());
Assert.Equal(r, messagePredicate + i);
Assert.Equal(0, consumer.NumMessagesInQueue());
_output.WriteLine("Consumer received : " + r);
}
}
public void TestPauseAndResume()
{
const string topicName = "zero-queue-pause-and-resume";
const string subName = "sub";
AtomicReference <CountdownEvent> latch = new AtomicReference <CountdownEvent>(new CountdownEvent(1));
AtomicInteger received = new AtomicInteger();
var config = new ConsumerConfigBuilder <sbyte[]>()
.Topic(topicName)
.SubscriptionName(subName)
.ReceiverQueueSize(0)
.MessageListener(new MessageListener <sbyte[]>((consumer, msg) =>
{
Assert.NotNull(msg);
consumer.Tell(new AcknowledgeMessage <sbyte[]>(msg));
received.GetAndIncrement();
latch.Value.AddCount();
}, null));
var consumer = _client.NewConsumer(config);
consumer.Pause();
var pBuilder = new ProducerConfigBuilder <sbyte[]>()
.Topic(topicName)
.EnableBatching(false);
Producer <sbyte[]> producer = _client.NewProducer(pBuilder);
for (int i = 0; i < 2; i++)
{
producer.Send(Encoding.UTF8.GetBytes("my-message-" + i).ToSBytes());
}
// Paused consumer receives only one message
//Assert.True(latch.Value.Wait(TimeSpan.FromSeconds(2)));
//Thread.Sleep(2000);
//Assert.Equal(1, received.GetValue());
//latch.GetAndSet(new CountdownEvent(1));
consumer.Resume();
Thread.Sleep(10000);
//Assert.True(latch.Value.Wait(TimeSpan.FromSeconds(2)), "Timed out waiting for message listener acks");
consumer.Unsubscribe();
producer.Close();
}
private void PlainKeyValueProducer(string topic)
{
//var jsonSchem = AvroSchema<JournalEntry>.Of(typeof(JournalEntry));
var jsonSchem = KeyValueSchema <string, string> .Of(ISchema <string> .String, ISchema <string> .String);
var builder = new ConsumerConfigBuilder <KeyValue <string, string> >()
.Topic(topic)
.SubscriptionName($"subscriber-name-{DateTimeHelper.CurrentUnixTimeMillis()}")
.AckTimeout(TimeSpan.FromMilliseconds(20000))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(jsonSchem, builder);
var producerConfig = new ProducerConfigBuilder <KeyValue <string, string> >()
.ProducerName(topic.Split("/").Last())
.Topic(topic)
.Schema(jsonSchem)
.SendTimeout(10000);
var producer = _client.NewProducer(jsonSchem, producerConfig);
for (var i = 0; i < 10; i++)
{
var metadata = new Dictionary <string, string>
{
["Key"] = "Single",
["Properties"] = JsonSerializer.Serialize(new Dictionary <string, string> {
{ "Tick", DateTime.Now.Ticks.ToString() }
}, new JsonSerializerOptions {
WriteIndented = true
})
};
var id = producer.NewMessage().Properties(metadata).Value <string, string>(new KeyValue <string, string>("Ebere", $"[{i}]Ebere")).Send();
_output.WriteLine(id.ToString());
}
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < 10; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var kv = msg.Value;
_output.WriteLine($"key:{kv.Key}, value:{kv.Value}");
}
}
}
public void TestAddAndRemove()
{
var builder = new ConsumerConfigBuilder <byte[]>();
builder.Topic("TestAckTracker");
builder.SubscriptionName("TestAckTracker-sub");
var consumer = _client.NewConsumer(builder);
var unack = _system.ActorOf(UnAckedChunckedMessageIdSequenceMap.Prop());
var tracker = _client.ActorSystem.ActorOf(UnAckedMessageTracker.Prop(1000000, 100000, consumer.ConsumerActor, unack));
var empty = tracker.Ask <bool>(Empty.Instance).GetAwaiter().GetResult();
Assert.True(empty);
var size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(0, size);
var mid = new MessageId(1L, 1L, -1);
var added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult();
Assert.True(added);
added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult();
Assert.False(added);
size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(1, size);
tracker.Tell(Clear.Instance);
added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult();
Assert.True(added);
size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(1, size);
var removed = tracker.Ask <bool>(new Remove(mid)).GetAwaiter().GetResult();
Assert.True(removed);
empty = tracker.Ask <bool>(Empty.Instance).GetAwaiter().GetResult();
Assert.True(empty);
size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(0, size);
}
private static void MultiConsumer(PulsarClient client)
{
var messageCount = 5;
var first = $"one-topic-{Guid.NewGuid()}";
var second = $"two-topic-{Guid.NewGuid()}";
var third = $"three-topic-{Guid.NewGuid()}";
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(first, second, third)
.ForceTopicCreation(true)
.SubscriptionName("multi-topic-sub");
var consumer = client.NewConsumer(builder);
PublishMessages(first, messageCount, "hello Toba", client);
PublishMessages(third, messageCount, "hello Toba", client);
PublishMessages(second, messageCount, "hello Toba", client);
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
if (message != null)
{
consumer.Acknowledge(message);
Console.WriteLine($"message from topic: {message.TopicName}");
}
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
if (message != null)
{
consumer.Acknowledge(message);
Console.WriteLine($"message from topic: {message.TopicName}");
}
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive(); if (message != null)
{
consumer.Acknowledge(message);
Console.WriteLine($"message from topic: {message.TopicName}");
}
}
}
public void TestZeroQueueSizeMessageRedeliveryForListener()
{
string topic = $"testZeroQueueSizeMessageRedeliveryForListener-{DateTime.Now.Ticks}";
const int messages = 10;
CountdownEvent latch = new CountdownEvent(messages * 2);
ISet <int> receivedMessages = new HashSet <int>();
var config = new ConsumerConfigBuilder <int>()
.Topic(topic)
.SubscriptionName("sub")
.ReceiverQueueSize(0)
.SubscriptionType(SubType.Shared)
.AckTimeout(1, TimeUnit.SECONDS)
.MessageListener(new MessageListener <int>((consumer, msg) =>
{
try
{
receivedMessages.Add(msg.Value);
}
finally
{
latch.Signal();
}
}, null));
var consumer = _client.NewConsumer(ISchema <object> .Int32, config);
var pBuilder = new ProducerConfigBuilder <int>()
.Topic(topic)
.EnableBatching(false);
var producer = _client.NewProducer(ISchema <object> .Int32, pBuilder);
for (int i = 0; i < messages; i++)
{
producer.Send(i);
}
latch.Wait();
Assert.Equal(receivedMessages.Count, messages);
consumer.Close();
producer.Close();
}
public void ProduceAndConsume()
{
var topic = $"persistent://public/default/my-topic-{Guid.NewGuid()}";
var r = new Random(0);
var byteKey = new byte[1000];
r.NextBytes(byteKey);
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.ForceTopicCreation(true)
.SubscriptionName($"ByteKeysTest-subscriber-{Guid.NewGuid()}");
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>();
producerBuilder.Topic(topic);
var producer = _client.NewProducer(producerBuilder);
producer.NewMessage().KeyBytes(byteKey)
.Properties(new Dictionary <string, string> {
{ "KeyBytes", Encoding.UTF8.GetString(byteKey) }
})
.Value(Encoding.UTF8.GetBytes("TestMessage"))
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var message = consumer.Receive();
Assert.Equal(byteKey, message.KeyBytes);
Assert.True(message.HasBase64EncodedKey());
var receivedMessage = Encoding.UTF8.GetString(message.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.Equal("TestMessage", receivedMessage);
producer.Close();
consumer.Close();
}
private IActorRef Consumer(PulsarSettings pulsarSettings)
{
var consumerListener = new DefaultConsumerEventListener(l => { });
var consumerConfig = new ConsumerConfigBuilder()
.ConsumerName(pulsarSettings.Topic)
.ForceTopicCreation(true)
.SubscriptionName($"{pulsarSettings.Topic}-Subscription")
.Topic(pulsarSettings.Topic)
//.AckTimeout(10000)
.AcknowledgmentGroupTime(0)
.ConsumerEventListener(consumerListener)
.SubscriptionType(CommandSubscribe.SubType.Exclusive)
.Schema(_schema)
.SetConsumptionType(ConsumptionType.Queue)
.MessageListener(new DefaultMessageListener(null, null))
.StartMessageId(MessageIdFields.Earliest)
.SubscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
.ConsumerConfigurationData;
return(_pulsarSystem.PulsarConsumer(new CreateConsumer(_schema, consumerConfig)).Consumer);
}
public void ProduceAbortTest()
{
var txn = Txn;
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(_topicOutput)
.SubscriptionName($"test{DateTime.Now.Ticks}")
.ForceTopicCreation(true)
.EnableBatchIndexAcknowledgment(true)
.SubscriptionInitialPosition(SubscriptionInitialPosition.Earliest);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>();
producerBuilder.Topic(_topicOutput);
producerBuilder.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var messageCnt = 10;
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage(txn).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
// Can't receive transaction messages before abort.
var message = consumer.Receive();
Assert.Null(message);
txn.Abort();
// Cant't receive transaction messages after abort.
message = consumer.Receive();
Assert.Null(message);
}
public void TestMultiTopicConsumer()
{
var messageCount = 5;
var first = $"one-topic-{Guid.NewGuid()}";
var second = $"two-topic-{Guid.NewGuid()}";
var third = $"three-topic-{Guid.NewGuid()}";
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(first, second, third)
.ForceTopicCreation(true)
.SubscriptionName("multi-topic-sub");
var consumer = _client.NewConsumer(builder);
PublishMessages(first, messageCount, "hello Toba");
PublishMessages(third, messageCount, "hello Toba");
PublishMessages(second, messageCount, "hello Toba");
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
_output.WriteLine($"message from topic: {message.Topic}");
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
Assert.NotNull(message);
_output.WriteLine($"message from topic: {message.Topic}");
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
Assert.NotNull(message);
_output.WriteLine($"message from topic: {message.Topic}");
}
consumer.Close();
}
public void TestZeroQueueSizeMessageRedelivery()
{
const string topic = "testZeroQueueSizeMessageRedelivery";
var config = new ConsumerConfigBuilder <int>()
.Topic(topic)
.SubscriptionName("sub")
.ReceiverQueueSize(0)
.SubscriptionType(SubType.Shared)
.AckTimeout(1, TimeUnit.SECONDS);
var consumer = _client.NewConsumer(ISchema <object> .Int32, config);
var pBuilder = new ProducerConfigBuilder <int>()
.Topic(topic)
.EnableBatching(false);
var producer = _client.NewProducer(ISchema <object> .Int32, pBuilder);
const int messages = 10;
for (int i = 0; i < messages; i++)
{
producer.Send(i);
}
ISet <int> receivedMessages = new HashSet <int>();
for (int i = 0; i < messages * 2; i++)
{
receivedMessages.Add(consumer.Receive().Value);
}
Assert.Equal(receivedMessages.Count, messages);
consumer.Close();
producer.Close();
}
public void TestAckTracker()
{
var builder = new ConsumerConfigBuilder <byte[]>();
builder.AcknowledgmentGroupTime(10000);
builder.Topic($"TestAckTracker-{Guid.NewGuid()}");
builder.SubscriptionName($"TestAckTracker-sub-{Guid.NewGuid()}");
var conf = builder.ConsumerConfigurationData;
var consumer = _client.NewConsumer(builder);
var unack = _system.ActorOf(UnAckedChunckedMessageIdSequenceMap.Prop());
var tracker = _client.ActorSystem.ActorOf(PersistentAcknowledgmentsGroupingTracker <byte[]> .Prop(unack, consumer.ConsumerActor, consumer.ConsumerActor /*dummy*/, 1, consumer.ConsumerActor, conf));
var msg1 = new MessageId(5, 1, 0);
var msg2 = new MessageId(5, 2, 0);
var msg3 = new MessageId(5, 3, 0);
var msg4 = new MessageId(5, 4, 0);
var msg5 = new MessageId(5, 5, 0);
var msg6 = new MessageId(5, 6, 0);
var isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg1, CommandAck.AckType.Individual, new Dictionary <string, long>()));
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg5, CommandAck.AckType.Cumulative, new Dictionary <string, long>()));
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
// Flush while disconnected. the internal tracking will not change
tracker.Tell(FlushPending.Instance);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg6, CommandAck.AckType.Individual, new Dictionary <string, long>()));
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
tracker.Tell(FlushPending.Instance);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.GracefulStop(TimeSpan.FromSeconds(1));
}
public async ValueTask <Consumer <T> > NewConsumerAsync <T>(ISchema <T> schema, ConsumerConfigBuilder <T> confBuilder)
{
var conf = confBuilder.ConsumerConfigurationData;
// DLQ only supports non-ordered subscriptions, don't enable DLQ on Key_Shared subType since it require message ordering for given key.
if (conf.SubscriptionType == SubType.KeyShared && !string.IsNullOrWhiteSpace(conf.DeadLetterPolicy?.DeadLetterTopic))
{
throw new PulsarClientException.InvalidConfigurationException("Deadletter topic on Key_Shared " +
"subscription type is not supported.");
}
if (conf.TopicNames.Count == 0 && conf.TopicsPattern == null)
{
throw new PulsarClientException.InvalidConfigurationException("Topic name must be set on the consumer builder");
}
if (string.IsNullOrWhiteSpace(conf.SubscriptionName))
{
throw new PulsarClientException.InvalidConfigurationException("Subscription name must be set on the consumer builder");
}
if (conf.KeySharedPolicy != null && conf.SubscriptionType != CommandSubscribe.SubType.KeyShared)
{
throw new PulsarClientException.InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription");
}
if (conf.RetryEnable && conf.TopicNames.Count > 0)
{
var topicFirst = TopicName.Get(conf.TopicNames.GetEnumerator().Current);
var retryLetterTopic = topicFirst.Namespace + "/" + conf.SubscriptionName + RetryMessageUtil.RetryGroupTopicSuffix;
var deadLetterTopic = topicFirst.Namespace + "/" + conf.SubscriptionName + RetryMessageUtil.DlqGroupTopicSuffix;
if (conf.DeadLetterPolicy == null)
{
conf.DeadLetterPolicy = new DeadLetterPolicy
{
MaxRedeliverCount = RetryMessageUtil.MaxReconsumetimes,
RetryLetterTopic = retryLetterTopic,
DeadLetterTopic = deadLetterTopic
};
}
else
{
if (string.IsNullOrWhiteSpace(conf.DeadLetterPolicy.RetryLetterTopic))
{
conf.DeadLetterPolicy.RetryLetterTopic = retryLetterTopic;
}
if (string.IsNullOrWhiteSpace(conf.DeadLetterPolicy.DeadLetterTopic))
{
conf.DeadLetterPolicy.DeadLetterTopic = deadLetterTopic;
}
}
conf.TopicNames.Add(conf.DeadLetterPolicy.RetryLetterTopic);
}
return(await Subscribe(conf, schema).ConfigureAwait(false));
}
public void TestGenericTopic()
{
var schema = AvroSchema <ComplexGenericData> .Of(typeof(ComplexGenericData));
var genericSchema = GenericAvroSchema.Of(schema.SchemaInfo);
_output.WriteLine(schema.SchemaInfo.SchemaDefinition);
var pBuilder = new ProducerConfigBuilder <IGenericRecord>()
.Topic(_topic);
var producer = _client.NewProducer(genericSchema, pBuilder);
const int messageCount = 10;
for (var i = 0; i < messageCount; i++)
{
var dataForWriter = new GenericRecord((Avro.RecordSchema)genericSchema.AvroSchema);
dataForWriter.Add("Feature", "Education");
dataForWriter.Add("StringData", new Dictionary <string, string> {
{ "Index", i.ToString() }, { "FirstName", "Ebere" }, { "LastName", "Abanonu" }
});
dataForWriter.Add("ComplexData", ToBytes(new ComplexData {
ProductId = i, Point = i * 2, Sales = i * 2 * 5
}));
var record = new GenericAvroRecord(null, genericSchema.AvroSchema, genericSchema.Fields, dataForWriter);
var receipt = producer.Send(record);
_output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions {
WriteIndented = true
}));
}
var messageReceived = 0;
var builder = new ConsumerConfigBuilder <IGenericRecord>()
.Topic(_topic)
.ForceTopicCreation(true)
.SubscriptionName($"generic_sub");
var consumer = _client.NewConsumer(ISchema <object> .AutoConsume(), builder);
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; ++i)
{
var m = consumer.Receive();
Assert.NotNull(m);
var receivedMessage = m.Value;
var feature = receivedMessage.GetField("Feature").ToString();
var strinData = (Dictionary <string, object>)receivedMessage.GetField("StringData");
var complexData = FromBytes <ComplexData>((byte[])receivedMessage.GetField("ComplexData"));
_output.WriteLine(feature);
_output.WriteLine(JsonSerializer.Serialize(strinData, new JsonSerializerOptions {
WriteIndented = true
}));
_output.WriteLine(JsonSerializer.Serialize(complexData, new JsonSerializerOptions {
WriteIndented = true
}));
messageReceived++;
consumer.Acknowledge(m);
}
Assert.Equal(10, messageReceived);
producer.Close();
consumer.Close();
}
public void TxnAckTestBatchedFailoverSub()
{
var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(normalTopic)
.SubscriptionName($"test-{Guid.NewGuid()}")
.ForceTopicCreation(true)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(5000)
.SubscriptionType(SubType.Failover);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(normalTopic)
.EnableBatching(true)
.BatchingMaxMessages(100);
var producer = _client.NewProducer(producerBuilder);
for (var retryCnt = 0; retryCnt < 1; retryCnt++)
{
var txn = Txn;
//Thread.Sleep(TimeSpan.FromSeconds(30));
var messageCnt = 100;
// produce normal messages
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage().Value("hello".GetBytes()).Send();
}
// consume and ack messages with txn
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
var msg = consumer.Receive();
Assert.NotNull(msg);
_output.WriteLine($"receive msgId: {msg.MessageId}, count : {i}");
consumer.Acknowledge(msg.MessageId, txn);
}
// the messages are pending ack state and can't be received
var message = consumer.Receive();
Assert.Null(message);
// 1) txn abort
txn.Abort();
// after transaction abort, the messages could be received
var commitTxn = Txn;
//Thread.Sleep(TimeSpan.FromSeconds(5));
Thread.Sleep(TimeSpan.FromSeconds(30));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message.MessageId, commitTxn);
_output.WriteLine($"receive msgId: {message.MessageId}, count: {i}");
}
// 2) ack committed by a new txn
commitTxn.Commit();
// after transaction commit, the messages can't be received
message = consumer.Receive();
Assert.Null(message);
}
}
private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout)
{
_output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}");
var topic = "testNegativeAcks-" + DateTime.Now.Ticks;
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName($"sub1-{Guid.NewGuid()}")
.AckTimeout(TimeSpan.FromMilliseconds(ackTimeout))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0)
.NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis))
.SubscriptionType(subscriptionType);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
if (batching)
{
pBuilder.EnableBatching(batching);
pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis));
pBuilder.BatchingMaxMessages(10);
}
var producer = _client.NewProducer(pBuilder);
ISet <string> sentMessages = new HashSet <string>();
const int n = 10;
for (var i = 0; i < n; i++)
{
var value = "test-" + i;
producer.Send(Encoding.UTF8.GetBytes(value));
sentMessages.Add(value);
}
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
consumer.NegativeAcknowledge(msg);
_output.WriteLine(ms);
}
}
ISet <string> receivedMessages = new HashSet <string>();
Thread.Sleep(TimeSpan.FromSeconds(10));
// All the messages should be received again
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
_output.WriteLine(ms);
receivedMessages.Add(ms);
consumer.Acknowledge(msg);
}
}
Assert.Equal(sentMessages, receivedMessages);
var nu = consumer.Receive();
// There should be no more messages
Assert.Null(nu);
producer.Close();
consumer.Close();
}
public void TxnCumulativeAckTestBatched()
{
var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(normalTopic)
.ForceTopicCreation(true)
.SubscriptionName($"test-{Guid.NewGuid()}")
.EnableBatchIndexAcknowledgment(true)
.SubscriptionType(SubType.Failover)
.AcknowledgmentGroupTime(3000)
.AckTimeout(TimeSpan.FromMilliseconds(10000));
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(normalTopic)
.EnableBatching(true)
.BatchingMaxMessages(50)
.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(1000));
var producer = _client.NewProducer(producerBuilder);
for (var retryCnt = 0; retryCnt < 2; retryCnt++)
{
var abortTxn = Txn;
var messageCnt = 100;
// produce normal messages
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage().Value(Encoding.UTF8.GetBytes("Hello")).Send();
}
IMessage <byte[]> message = null;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
if (i % 3 == 0)
{
// throws org.apache.pulsar.transaction.common.exception.TransactionCon"org.apache.pulsar.transaction.common.exception.TransactionConflictException: [persistent://public/default/normal-topic-24636acf-51a4-4309-8f68-86354383cefe][test-b1954d51-2e93-49f9-a3b7-2f76dcdedd36] Transaction:(1,42) try to cumulative batch ack position: 14960:0 within range of current currentPosition: 14960:0
//better done outside
consumer.AcknowledgeCumulative(message.MessageId, abortTxn);
}
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
// consumer.AcknowledgeCumulative(message.MessageId, abortTxn);
// the messages are pending ack state and can't be received
message = consumer.Receive();
Assert.Null(message);
abortTxn.Abort();
var commitTxn = Txn;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
if (i % 3 == 0)
{
consumer.AcknowledgeCumulative(message.MessageId, commitTxn);
}
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
commitTxn.Commit();
Thread.Sleep(TimeSpan.FromSeconds(5));
message = consumer.Receive();
Assert.Null(message);
}
}
public void TxnCumulativeAckTest()
{
var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(normalTopic)
.SubscriptionName($"test-{Guid.NewGuid()}")
.ForceTopicCreation(true)
//.SubscriptionType(SubType.Failover)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(3000)
.AckTimeout(TimeSpan.FromMilliseconds(10000));
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(normalTopic);
var producer = _client.NewProducer(producerBuilder);
for (var retryCnt = 0; retryCnt < 2; retryCnt++)
{
var abortTxn = Txn;
var messageCnt = 50;
// produce normal messages
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage().Value(Encoding.UTF8.GetBytes("Hello")).Send();
}
IMessage <byte[]> message = null;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
if (i % 3 == 0)
{
consumer.AcknowledgeCumulative(message.MessageId, abortTxn);
}
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
// the messages are pending ack state and can't be received
message = consumer.Receive();
Assert.Null(message);
abortTxn.Abort();
var commitTxn = Txn;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.AcknowledgeCumulative(message.MessageId, commitTxn);
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
commitTxn.Commit();
Thread.Sleep(TimeSpan.FromSeconds(5));
message = consumer.Receive();
Assert.Null(message);
}
}
public void TxnMessageAckTest()
{
var topic = $"{_topicMessageAckTest}-{Guid.NewGuid()}";
var subName = $"test-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName(subName)
.ForceTopicCreation(true)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.EnableBatching(false)
.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var txn = Txn;
var messageCnt = 10;
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage(txn).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
_output.WriteLine("produce transaction messages finished");
// Can't receive transaction messages before commit.
var message = consumer.Receive();
Assert.Null(message);
_output.WriteLine("transaction messages can't be received before transaction committed");
txn.Commit();
var ackedMessageCount = 0;
var receiveCnt = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
receiveCnt++;
if (i % 2 == 0)
{
consumer.Acknowledge(message);
ackedMessageCount++;
}
}
Assert.Equal(messageCnt, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
consumer.RedeliverUnacknowledgedMessages();
Thread.Sleep(TimeSpan.FromSeconds(10));
receiveCnt = 0;
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
receiveCnt++;
}
Assert.Equal(messageCnt - ackedMessageCount, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
_output.WriteLine($"receive transaction messages count: {receiveCnt}");
}
private void PlainAvroProducer(string topic)
{
var jsonSchem = AvroSchema <JournalEntry> .Of(typeof(JournalEntry));
var builder = new ConsumerConfigBuilder <JournalEntry>()
.Topic(topic)
.SubscriptionName($"my-subscriber-name-{DateTimeHelper.CurrentUnixTimeMillis()}")
.AckTimeout(TimeSpan.FromMilliseconds(20000))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(jsonSchem, builder);
var producerConfig = new ProducerConfigBuilder <JournalEntry>()
.ProducerName(topic.Split("/").Last())
.Topic(topic)
.Schema(jsonSchem)
.SendTimeout(10000);
var producer = _client.NewProducer(jsonSchem, producerConfig);
for (var i = 0; i < 10; i++)
{
var student = new Students
{
Name = $"[{i}] Ebere: {DateTimeOffset.Now.ToUnixTimeMilliseconds()} - presto-ed {DateTime.Now.ToString(CultureInfo.InvariantCulture)}",
Age = 202 + i,
School = "Akka-Pulsar university"
};
var journal = new JournalEntry
{
Id = $"[{i}]Ebere: {DateTimeOffset.Now.ToUnixTimeMilliseconds()}",
PersistenceId = "sampleActor",
IsDeleted = false,
Ordering = 0,
Payload = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(student)),
SequenceNr = 0,
Tags = "root"
};
var metadata = new Dictionary <string, string>
{
["Key"] = "Single",
["Properties"] = JsonSerializer.Serialize(new Dictionary <string, string> {
{ "Tick", DateTime.Now.Ticks.ToString() }
}, new JsonSerializerOptions {
WriteIndented = true
})
};
var id = producer.NewMessage().Properties(metadata).Value(journal).Send();
}
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < 10; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var receivedMessage = msg.Value;
_output.WriteLine(JsonSerializer.Serialize(receivedMessage, new JsonSerializerOptions {
WriteIndented = true
}));
}
}
}
public Consumer <T> NewConsumer <T>(ISchema <T> schema, ConsumerConfigBuilder <T> confBuilder)
{
return(NewConsumerAsync(schema, confBuilder).GetAwaiter().GetResult());
}
public void ProduceCommitTest()
{
var txn1 = Txn;
var txn2 = Txn;
var topic = $"{_topicOutput}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.ForceTopicCreation(true)
.SubscriptionName($"test-{Guid.NewGuid()}");
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var txnMessageCnt = 0;
var messageCnt = 10;
for (var i = 0; i < messageCnt; i++)
{
if (i % 5 == 0)
{
producer.NewMessage(txn1).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
else
{
producer.NewMessage(txn2).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
txnMessageCnt++;
}
// Can't receive transaction messages before commit.
var message = consumer.Receive();
Assert.Null(message);
txn1.Commit();
_output.WriteLine($"Committed 1");
txn2.Commit();
_output.WriteLine($"Committed 1");
// txn1 messages could be received after txn1 committed
var receiveCnt = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < txnMessageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
_output.WriteLine(Encoding.UTF8.GetString(message.Value));
receiveCnt++;
}
for (var i = 0; i < txnMessageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
receiveCnt++;
}
Assert.Equal(txnMessageCnt, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
_output.WriteLine($"message commit test enableBatch {true}");
}
