private ISet <string> PublishMessages(string topic, int count, bool enableBatch)
{
ISet <string> keys = new HashSet <string>();
var builder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.MessageRoutingMode(Common.MessageRoutingMode.RoundRobinMode)
.MaxPendingMessages(count)
.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(800000));
if (enableBatch)
{
builder.EnableBatching(true);
builder.BatchingMaxMessages(count);
}
else
{
builder.EnableBatching(false);
}
var producer = _client.NewProducer(builder);
for (var i = 0; i < count; i++)
{
var key = "key" + i;
var data = Encoding.UTF8.GetBytes("my-message-" + i);
producer.NewMessage().Key(key).Value(data).Send();
keys.Add(key);
}
producer.Flush();
return(keys);
}
public void ProduceAndConsume()
{
var record1 = AvroSchema <SimpleRecord> .Of(typeof(SimpleRecord));
var consumerBuilder = new ConsumerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema1")
.SubscriptionName("test-sub");
var consumer = _client.NewConsumer(record1, consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema1");
var producer = _client.NewProducer(record1, producerBuilder);
producer.NewMessage()
.Value(new SimpleRecord {
Name = "Ebere Abanonu", Age = int.MaxValue
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
consumer.Unsubscribe();
var record2 = AvroSchema <SimpleRecord2> .Of(typeof(SimpleRecord2));
var consumerBuilder2 = new ConsumerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema2")
.SubscriptionName("test-sub");
var consumer1 = _client.NewConsumer(record2, consumerBuilder2);
var producerBuilder2 = new ProducerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema2");
var producer2 = _client.NewProducer(record2, producerBuilder2);
producer2.NewMessage()
.Value(new SimpleRecord2 {
Name = "Ebere", Age = int.MaxValue, Surname = "Abanonu"
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var msg = consumer1.Receive();
Assert.NotNull(msg);
consumer1.Acknowledge(msg);
consumer1.Unsubscribe();
producer.Close();
producer2.Close();
consumer.Close();
consumer1.Close();
}
public void TestMultiTopic()
{
var topic = "persistent://public/default/topic" + Guid.NewGuid();
var topic2 = "persistent://public/default/topic2" + Guid.NewGuid();
var topic3 = "persistent://public/default/topic3" + Guid.NewGuid();
IList <string> topics = new List <string> {
topic, topic2, topic3
};
var builder = new ReaderConfigBuilder <string>()
.Topics(topics)
.StartMessageId(IMessageId.Earliest)
.ReaderName("my-reader");
var reader = _client.NewReader(ISchema <object> .String, builder);
// create producer and send msg
IList <Producer <string> > producerList = new List <Producer <string> >();
foreach (var topicName in topics)
{
var producer = _client.NewProducer(ISchema <object> .String, new ProducerConfigBuilder <string>().Topic(topicName));
producerList.Add(producer);
}
var msgNum = 10;
ISet <string> messages = new HashSet <string>();
for (var i = 0; i < producerList.Count; i++)
{
var producer = producerList[i];
for (var j = 0; j < msgNum; j++)
{
var msg = i + "msg" + j;
producer.Send(msg);
messages.Add(msg);
}
}
// receive messagesS
var message = reader.ReadNext(TimeSpan.FromSeconds(5));
while (message != null)
{
var value = message.Value;
_output.WriteLine(value);
Assert.True(messages.Remove(value));
message = reader.ReadNext(TimeSpan.FromSeconds(5));
}
Assert.Equal(0, messages.Count);
// clean up
foreach (var producer in producerList)
{
producer.Close();
}
reader.Close();
}
public virtual void ProducerInstantiation()
{
var producer = new ProducerConfigBuilder <string>();
producer.Topic("ProducerInstantiation");
var stringProducerBuilder = _client.NewProducer(new StringSchema(), producer);
Assert.NotNull(stringProducerBuilder);
stringProducerBuilder.Close();
}
private ISet <MessageId> PublishMessages(string topic, int count)
{
var ids = new HashSet <MessageId>();
var builder = new ProducerConfigBuilder <DataOp>()
.Topic(topic);
var producer = _client.NewProducer(AvroSchema <DataOp> .Of(typeof(DataOp)), builder);
for (var i = 0; i < count; i++)
{
var key = "key" + i;
MessageId id = null;
if (i % 2 == 0)
{
id = producer.NewMessage().Key(key).Property("twitter", "mestical").Value(new DataOp {
Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis()
}).Send();
}
else
{
id = producer.NewMessage().Key(key).Value(new DataOp {
Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis()
}).Send();
}
ids.Add(id);
}
return(ids);
}
private static void ExclusiveProduceConsumer(PulsarClient pulsarClient)
{
var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>()
.AccessMode(SharpPulsar.Common.ProducerAccessMode.Exclusive)
.Topic(myTopic));;
for (var i = 0; i < 100; i++)
{
var data = Encoding.UTF8.GetBytes($"tuts-{i}");
producer.NewMessage().Value(data).Send();
}
var pool = ArrayPool <byte> .Shared;
var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(myTopic)
.ForceTopicCreation(true)
.SubscriptionName("myTopic-sub-Exclusive")
.SubscriptionInitialPosition(SharpPulsar.Common.SubscriptionInitialPosition.Earliest));
for (var i = 0; i < 100; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"message '{res}' from topic: {message.TopicName}");
}
}
}
private ISet <string> PublishMessages(string topic, int count)
{
ISet <string> keys = new HashSet <string>();
var builder = new ProducerConfigBuilder <DataOp>()
.Topic(topic);
var producer = _client.NewProducer(AvroSchema <DataOp> .Of(typeof(DataOp)), builder);
for (var i = 0; i < count; i++)
{
var key = "key" + i;
producer.NewMessage().Key(key).Value(new DataOp {
Text = "my-sql-message-" + i
}).Send();
keys.Add(key);
}
return(keys);
}
public void ZeroQueueSizeNormalConsumer()
{
string key = "nonZeroQueueSizeNormalConsumer";
// 1. Config
string topicName = "topic-" + key;
string subscriptionName = "my-ex-subscription-" + key;
string messagePredicate = "my-message-" + key + "-";
// 2. Create Producer
var pBuilder = new ProducerConfigBuilder <sbyte[]>()
.Topic(topicName)
.EnableBatching(false);
var producer = _client.NewProducer(pBuilder);
// 3. Create Consumer
var config = new ConsumerConfigBuilder <sbyte[]>()
.Topic(topicName)
.SubscriptionName(subscriptionName)
.ReceiverQueueSize(0);
var consumer = _client.NewConsumer(config);
// 3. producer publish messages
for (int i = 0; i < _totalMessages; i++)
{
var msg = messagePredicate + i;
_output.WriteLine("Producer produced: " + msg);
producer.Send(Encoding.UTF8.GetBytes(msg).ToSBytes());
}
// 4. Receiver receives the message
IMessage <sbyte[]> message;
for (int i = 0; i < _totalMessages; i++)
{
Assert.Equal(0, consumer.NumMessagesInQueue());
message = consumer.Receive();
var r = Encoding.UTF8.GetString(message.Data.ToBytes());
Assert.Equal(r, messagePredicate + i);
Assert.Equal(0, consumer.NumMessagesInQueue());
_output.WriteLine("Consumer received : " + r);
}
}
public void TestUnAckMessageRedeliveryWithReceive()
{
var topic = $"persistent://public/default/async-unack-redelivery-{Guid.NewGuid()}";
var builder = new ConsumerConfigBuilder <byte[]>();
builder.Topic(topic);
builder.SubscriptionName("sub-TestUnAckMessageRedeliveryWithReceive");
builder.AckTimeout(TimeSpan.FromMilliseconds(8000));
builder.ForceTopicCreation(true);
builder.AcknowledgmentGroupTime(0);
builder.SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
var producer = _client.NewProducer(pBuilder);
const int messageCount = 10;
for (var i = 0; i < messageCount; i++)
{
var receipt = producer.Send(Encoding.UTF8.GetBytes("my-message-" + i));
_output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions {
WriteIndented = true
}));
}
var messageReceived = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; ++i)
{
var m = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(m.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.NotNull(receivedMessage);
messageReceived++;
}
Assert.Equal(10, messageReceived);
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < messageCount; i++)
{
var m = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(m.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.NotNull(receivedMessage);
messageReceived++;
}
Assert.Equal(20, messageReceived);
producer.Close();
consumer.Close();
}
private static void Transaction(PulsarClient pulsarClient)
{
var txn = (Transaction)pulsarClient.NewTransaction().WithTransactionTimeout(TimeSpan.FromMinutes(5)).Build();
var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>()
.SendTimeout(0)
.Topic(myTopic));
var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(myTopic)
.ForceTopicCreation(true)
.SubscriptionName("myTopic-sub"));
for (var i = 0; i < 10; i++)
{
var text = $"tuts-{i}";
var data = Encoding.UTF8.GetBytes(text);
producer.NewMessage(txn).Value(data).Send();
Console.WriteLine($"produced: {text}");
}
var pool = ArrayPool <byte> .Shared;
//Should not consume messages as the transaction is not committed yet
for (var i = 0; i < 10; ++i)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"[1] message '{res}' from topic: {message.TopicName}");
}
}
txn.Commit();
for (var i = 0; i < 10; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"[2] message '{res}' from topic: {message.TopicName}");
}
}
}
public void TestDeliverAfter()
{
var numMessages = 5;
var consumer = _client.NewConsumer(ISchema <string> .String, new ConsumerConfigBuilder <string>()
.Topic(_topic)
.SubscriptionName($"delayed-sub-{Guid.NewGuid()}")
//deliverat works with shared subscription
.SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared)
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
var producer = _client.NewProducer(ISchema <string> .String, new ProducerConfigBuilder <string>()
.Topic(_topic));
// delay 5 seconds using DeliverAfter
for (var i = 0; i < numMessages; i++)
{
producer.NewMessage().Value("DeliverAfter message " + i).DeliverAfter(TimeSpan.FromMilliseconds(5000)).Send();
}
producer.Flush();
var numReceived = 0;
while (numMessages <= 0 || numReceived < numMessages)
{
var msg = consumer.Receive();
if (msg == null)
{
Thread.Sleep(TimeSpan.FromSeconds(1));
continue;
}
var dt = (DateTimeHelper.CurrentUnixTimeMillis() - msg.PublishTime) / 1000;
_output.WriteLine("Consumer Received message : " + msg.Value + "; Difference between publish time and receive time = " + dt + " seconds");
consumer.Acknowledge(msg);
++numReceived;
}
_output.WriteLine("Successfully received " + numReceived + " messages");
producer.Close();
consumer.Close();
}
public void A_100_000_producers()
{
for (var i = 0; i < 100_000; i++)
{
var p = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic($"A_100_000_producers-{i}-{Guid.NewGuid()}"));
var data = Encoding.UTF8.GetBytes($"A_100_000_producers-{i}");
var id = p.NewMessage().Value(data).Send();
Console.WriteLine($"A_100_000_producers({id.LedgerId}:{id.EntryId})");
}
Console.WriteLine("100,000 producers");
}
public void TestLargeMessage()
{
//this.conf.MaxMessageSize = 5;
const int totalMessages = 3;
var topicName = $"persistent://public/default/my-topic1-{DateTimeHelper.CurrentUnixTimeMillis()}";
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(topicName)
.SubscriptionName("my-subscriber-name")
.AckTimeout(TimeSpan.FromMilliseconds(20000))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topicName)
.EnableChunking(true)
.MaxMessageSize(5);
var producer = _client.NewProducer(pBuilder);
IList <string> publishedMessages = new List <string>();
for (var i = 1; i < totalMessages; i++)
{
var message = CreateMessagePayload(i * 10);
publishedMessages.Add(message);
producer.Send(Encoding.UTF8.GetBytes(message));
}
IMessage <byte[]> msg = null;
ISet <string> messageSet = new HashSet <string>();
IList <IMessage <byte[]> > msgIds = new List <IMessage <byte[]> >();
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < totalMessages - 1; i++)
{
msg = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(msg.Data);
_output.WriteLine($"[{i}] - Published [{publishedMessages[i]}] Received message: [{receivedMessage}]");
var expectedMessage = publishedMessages[i];
TestMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
msgIds.Add(msg);
}
foreach (var msgId in msgIds)
{
consumer.Acknowledge(msgId);
}
producer.Close();
consumer.Close();
}
public void Produce_Consumer_Allocation()
{
var topic = $"memory-allocation-{Guid.NewGuid()}";
var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName($"myTopic-sub-{Guid.NewGuid()}")
.IsAckReceiptEnabled(true)
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topic));
for (var i = 0; i < 500; i++)
{
var data = Encoding.UTF8.GetBytes($"memory-allocation-{i}");
producer.NewMessage().Value(data).Send();
}
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < 500; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
_output.WriteLine($"message '{res}' from topic: {message.Topic}");
}
}
dotMemory.Check(memory =>
{
var result = memory
.GetObjects(where => @where.Namespace.Like("SharpPulsar.*"));
_output.WriteLine(
ConsoleTable.From(
result
.GroupByType()
.Select(t => new
{
t.Type.Name,
t.ObjectsCount,
t.SizeInBytes
})
.OrderByDescending(x => x.SizeInBytes)
).ToMarkDownString());
Assert.True(result.ObjectsCount > 0);
});
}
static async Task Publish()
{
var producer = await client.NewProducer(Schema.JSON <Test>())
.Topic(topicName)
.CreateAsync();
while (running)
{
await producer.SendAsync(new Test { Text = DateTime.UtcNow.ToString() });
await Task.Delay(200);
}
//Console.WriteLine($"MessageId is: '{messageId}'");
}
private Producer <byte[]> CreateProducer(string topic, bool enableBatch, int batchSize = 500)
{
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
if (enableBatch)
{
pBuilder.EnableBatching(true);
pBuilder.BatchBuilder(IBatcherBuilder.KeyBased(_client.ActorSystem));
pBuilder.BatchingMaxMessages(batchSize);
pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(5000));
}
return(_client.NewProducer(pBuilder));
}
private static List <MessageId> PublishMessages(string topic, int count, string message, PulsarClient client)
{
var keys = new List <MessageId>();
var builder = new ProducerConfigBuilder <byte[]>()
.Topic(topic);
var producer = client.NewProducer(builder);
for (var i = 0; i < count; i++)
{
var key = "key" + i;
var data = Encoding.UTF8.GetBytes($"{message}-{i}");
var id = producer.NewMessage().Key(key).Value(data).Send();
keys.Add(id);
}
return(keys);
}
public void Setup()
{
var clientConfig = new PulsarClientConfigBuilder()
.ServiceUrl("pulsar://localhost:6650");
_pulsarSystem = PulsarSystem.GetInstance(clientConfig);
_client = _pulsarSystem.NewClient();
_consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(_benchTopic)
.ForceTopicCreation(true)
.SubscriptionName($"bench-sub-{Guid.NewGuid()}")
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
_producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(_benchTopic));
}
public void TestEncrptedProduceConsume()
{
var messageCount = 10;
var topic = $"encrypted-messages-{Guid.NewGuid()}";
var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.ForceTopicCreation(true)
.CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem"))
.SubscriptionName("encrypted-sub")
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem"))
.AddEncryptionKey("Ebere"));
for (var i = 0; i < messageCount; i++)
{
producer.Send(Encoding.UTF8.GetBytes($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]"));
}
var receivedCount = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; i++)
{
var message = consumer.Receive();
if (message != null)
{
var decrypted = Encoding.UTF8.GetString(message.Data);
_output.WriteLine(decrypted);
Assert.Equal($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]", decrypted);
consumer.Acknowledge(message);
receivedCount++;
}
}
Assert.True(receivedCount > 6);
producer.Close();
consumer.Close();
}
private static void ProduceConsumer(PulsarClient pulsarClient)
{
var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(myTopic)
.ForceTopicCreation(true)
.SubscriptionName($"sub-{Guid.NewGuid()}")
.IsAckReceiptEnabled(true));
var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(myTopic));
for (var i = 0; i < 1000; i++)
{
var data = Encoding.UTF8.GetBytes($"tuts-{i}");
var id = producer.NewMessage().Value(data).Send();
Console.WriteLine($"Message Id({id.LedgerId}:{id.EntryId})");
}
var pool = ArrayPool <byte> .Shared;
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < 1000; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"message '{res}' from topic: {message.TopicName}");
}
}
}
public void TestGenericTopic()
{
var schema = AvroSchema <ComplexGenericData> .Of(typeof(ComplexGenericData));
var genericSchema = GenericAvroSchema.Of(schema.SchemaInfo);
_output.WriteLine(schema.SchemaInfo.SchemaDefinition);
var pBuilder = new ProducerConfigBuilder <IGenericRecord>()
.Topic(_topic);
var producer = _client.NewProducer(genericSchema, pBuilder);
const int messageCount = 10;
for (var i = 0; i < messageCount; i++)
{
var dataForWriter = new GenericRecord((Avro.RecordSchema)genericSchema.AvroSchema);
dataForWriter.Add("Feature", "Education");
dataForWriter.Add("StringData", new Dictionary <string, string> {
{ "Index", i.ToString() }, { "FirstName", "Ebere" }, { "LastName", "Abanonu" }
});
dataForWriter.Add("ComplexData", ToBytes(new ComplexData {
ProductId = i, Point = i * 2, Sales = i * 2 * 5
}));
var record = new GenericAvroRecord(null, genericSchema.AvroSchema, genericSchema.Fields, dataForWriter);
var receipt = producer.Send(record);
_output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions {
WriteIndented = true
}));
}
var messageReceived = 0;
var builder = new ConsumerConfigBuilder <IGenericRecord>()
.Topic(_topic)
.ForceTopicCreation(true)
.SubscriptionName($"generic_sub");
var consumer = _client.NewConsumer(ISchema <object> .AutoConsume(), builder);
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; ++i)
{
var m = consumer.Receive();
Assert.NotNull(m);
var receivedMessage = m.Value;
var feature = receivedMessage.GetField("Feature").ToString();
var strinData = (Dictionary <string, object>)receivedMessage.GetField("StringData");
var complexData = FromBytes <ComplexData>((byte[])receivedMessage.GetField("ComplexData"));
_output.WriteLine(feature);
_output.WriteLine(JsonSerializer.Serialize(strinData, new JsonSerializerOptions {
WriteIndented = true
}));
_output.WriteLine(JsonSerializer.Serialize(complexData, new JsonSerializerOptions {
WriteIndented = true
}));
messageReceived++;
consumer.Acknowledge(m);
}
Assert.Equal(10, messageReceived);
producer.Close();
consumer.Close();
}
public void TxnAckTestBatchedFailoverSub()
{
var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(normalTopic)
.SubscriptionName($"test-{Guid.NewGuid()}")
.ForceTopicCreation(true)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(5000)
.SubscriptionType(SubType.Failover);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(normalTopic)
.EnableBatching(true)
.BatchingMaxMessages(100);
var producer = _client.NewProducer(producerBuilder);
for (var retryCnt = 0; retryCnt < 1; retryCnt++)
{
var txn = Txn;
//Thread.Sleep(TimeSpan.FromSeconds(30));
var messageCnt = 100;
// produce normal messages
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage().Value("hello".GetBytes()).Send();
}
// consume and ack messages with txn
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
var msg = consumer.Receive();
Assert.NotNull(msg);
_output.WriteLine($"receive msgId: {msg.MessageId}, count : {i}");
consumer.Acknowledge(msg.MessageId, txn);
}
// the messages are pending ack state and can't be received
var message = consumer.Receive();
Assert.Null(message);
// 1) txn abort
txn.Abort();
// after transaction abort, the messages could be received
var commitTxn = Txn;
//Thread.Sleep(TimeSpan.FromSeconds(5));
Thread.Sleep(TimeSpan.FromSeconds(30));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message.MessageId, commitTxn);
_output.WriteLine($"receive msgId: {message.MessageId}, count: {i}");
}
// 2) ack committed by a new txn
commitTxn.Commit();
// after transaction commit, the messages can't be received
message = consumer.Receive();
Assert.Null(message);
}
}
public void TxnCumulativeAckTest()
{
var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(normalTopic)
.SubscriptionName($"test-{Guid.NewGuid()}")
.ForceTopicCreation(true)
//.SubscriptionType(SubType.Failover)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(3000)
.AckTimeout(TimeSpan.FromMilliseconds(10000));
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(normalTopic);
var producer = _client.NewProducer(producerBuilder);
for (var retryCnt = 0; retryCnt < 2; retryCnt++)
{
var abortTxn = Txn;
var messageCnt = 50;
// produce normal messages
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage().Value(Encoding.UTF8.GetBytes("Hello")).Send();
}
IMessage <byte[]> message = null;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
if (i % 3 == 0)
{
consumer.AcknowledgeCumulative(message.MessageId, abortTxn);
}
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
// the messages are pending ack state and can't be received
message = consumer.Receive();
Assert.Null(message);
abortTxn.Abort();
var commitTxn = Txn;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.AcknowledgeCumulative(message.MessageId, commitTxn);
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
commitTxn.Commit();
Thread.Sleep(TimeSpan.FromSeconds(5));
message = consumer.Receive();
Assert.Null(message);
}
}
private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout)
{
_output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}");
var topic = "testNegativeAcks-" + DateTime.Now.Ticks;
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName($"sub1-{Guid.NewGuid()}")
.AckTimeout(TimeSpan.FromMilliseconds(ackTimeout))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0)
.NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis))
.SubscriptionType(subscriptionType);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
if (batching)
{
pBuilder.EnableBatching(batching);
pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis));
pBuilder.BatchingMaxMessages(10);
}
var producer = _client.NewProducer(pBuilder);
ISet <string> sentMessages = new HashSet <string>();
const int n = 10;
for (var i = 0; i < n; i++)
{
var value = "test-" + i;
producer.Send(Encoding.UTF8.GetBytes(value));
sentMessages.Add(value);
}
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
consumer.NegativeAcknowledge(msg);
_output.WriteLine(ms);
}
}
ISet <string> receivedMessages = new HashSet <string>();
Thread.Sleep(TimeSpan.FromSeconds(10));
// All the messages should be received again
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
_output.WriteLine(ms);
receivedMessages.Add(ms);
consumer.Acknowledge(msg);
}
}
Assert.Equal(sentMessages, receivedMessages);
var nu = consumer.Receive();
// There should be no more messages
Assert.Null(nu);
producer.Close();
consumer.Close();
}
public void TxnMessageAckTest()
{
var topic = $"{_topicMessageAckTest}-{Guid.NewGuid()}";
var subName = $"test-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName(subName)
.ForceTopicCreation(true)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.EnableBatching(false)
.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var txn = Txn;
var messageCnt = 10;
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage(txn).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
_output.WriteLine("produce transaction messages finished");
// Can't receive transaction messages before commit.
var message = consumer.Receive();
Assert.Null(message);
_output.WriteLine("transaction messages can't be received before transaction committed");
txn.Commit();
var ackedMessageCount = 0;
var receiveCnt = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
receiveCnt++;
if (i % 2 == 0)
{
consumer.Acknowledge(message);
ackedMessageCount++;
}
}
Assert.Equal(messageCnt, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
consumer.RedeliverUnacknowledgedMessages();
Thread.Sleep(TimeSpan.FromSeconds(10));
receiveCnt = 0;
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
receiveCnt++;
}
Assert.Equal(messageCnt - ackedMessageCount, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
_output.WriteLine($"receive transaction messages count: {receiveCnt}");
}
public void ProduceCommitTest()
{
var txn1 = Txn;
var txn2 = Txn;
var topic = $"{_topicOutput}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.ForceTopicCreation(true)
.SubscriptionName($"test-{Guid.NewGuid()}");
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var txnMessageCnt = 0;
var messageCnt = 10;
for (var i = 0; i < messageCnt; i++)
{
if (i % 5 == 0)
{
producer.NewMessage(txn1).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
else
{
producer.NewMessage(txn2).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
txnMessageCnt++;
}
// Can't receive transaction messages before commit.
var message = consumer.Receive();
Assert.Null(message);
txn1.Commit();
_output.WriteLine($"Committed 1");
txn2.Commit();
_output.WriteLine($"Committed 1");
// txn1 messages could be received after txn1 committed
var receiveCnt = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < txnMessageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
_output.WriteLine(Encoding.UTF8.GetString(message.Value));
receiveCnt++;
}
for (var i = 0; i < txnMessageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
receiveCnt++;
}
Assert.Equal(txnMessageCnt, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
_output.WriteLine($"message commit test enableBatch {true}");
}
public virtual void TestBinaryProtoToGetTopicsOfNamespacePersistent()
{
var key = Guid.NewGuid().ToString();
var subscriptionName = "regex-subscription";
var topicName1 = "persistent://public/default/reg-topic-1-" + key;
var topicName2 = "persistent://public/default/reg-topic-2-" + key;
var topicName3 = "persistent://public/default/reg-topic-3-" + key;
var topicName4 = "non-persistent://public/default/reg-topic-4-" + key;
var pattern = new Regex("public/default/reg-topic*");
// 2. create producer
var messagePredicate = "my-message-" + key + "-";
var producer1 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName1));
var producer2 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName2));
var producer3 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName3));
var producer4 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName4));
// 5. produce data
for (var i = 0; i < 10; i++)
{
producer1.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer1-" + i));
producer2.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer2-" + i));
producer3.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer3-" + i));
producer4.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer4-" + i));
}
// 6. should receive all the message
var messageSet = 0;
var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.TopicsPattern(pattern)
.PatternAutoDiscoveryPeriod(2)
.ForceTopicCreation(true)
.SubscriptionName(subscriptionName)
.SubscriptionType(SubType.Shared)
.AckTimeout(TimeSpan.FromMilliseconds(60000)));
Thread.Sleep(TimeSpan.FromSeconds(10));
var message = consumer.Receive();
if (message == null)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
message = consumer.Receive();
}
while (message != null)
{
var m = (TopicMessage <byte[]>)message;
messageSet++;
consumer.Acknowledge(m);
_output.WriteLine($"Consumer acknowledged : {Encoding.UTF8.GetString(message.Data)} from topic: {m.Topic}");
message = consumer.Receive();
}
consumer.Unsubscribe();
consumer.Close();
producer1.Close();
producer2.Close();
producer3.Close();
producer4.Close();
Assert.True(messageSet > 0);
}
