public void ProduceAndConsume()
{
var record1 = AvroSchema <SimpleRecord> .Of(typeof(SimpleRecord));
var consumerBuilder = new ConsumerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema1")
.SubscriptionName("test-sub");
var consumer = _client.NewConsumer(record1, consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema1");
var producer = _client.NewProducer(record1, producerBuilder);
producer.NewMessage()
.Value(new SimpleRecord {
Name = "Ebere Abanonu", Age = int.MaxValue
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
consumer.Unsubscribe();
var record2 = AvroSchema <SimpleRecord2> .Of(typeof(SimpleRecord2));
var consumerBuilder2 = new ConsumerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema2")
.SubscriptionName("test-sub");
var consumer1 = _client.NewConsumer(record2, consumerBuilder2);
var producerBuilder2 = new ProducerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema2");
var producer2 = _client.NewProducer(record2, producerBuilder2);
producer2.NewMessage()
.Value(new SimpleRecord2 {
Name = "Ebere", Age = int.MaxValue, Surname = "Abanonu"
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var msg = consumer1.Receive();
Assert.NotNull(msg);
consumer1.Acknowledge(msg);
consumer1.Unsubscribe();
producer.Close();
producer2.Close();
consumer.Close();
consumer1.Close();
}
public virtual void ConsumerInstantiation()
{
var consumer = new ConsumerConfigBuilder <string>();
consumer.Topic("ConsumerInstantiation");
consumer.SubscriptionName($"test-sub-{Guid.NewGuid()}");
var stringConsumerBuilder = _client.NewConsumer(new StringSchema(), consumer);
Assert.NotNull(stringConsumerBuilder);
stringConsumerBuilder.Close();
}
private static void ExclusiveProduceConsumer(PulsarClient pulsarClient)
{
var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>()
.AccessMode(SharpPulsar.Common.ProducerAccessMode.Exclusive)
.Topic(myTopic));;
for (var i = 0; i < 100; i++)
{
var data = Encoding.UTF8.GetBytes($"tuts-{i}");
producer.NewMessage().Value(data).Send();
}
var pool = ArrayPool <byte> .Shared;
var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(myTopic)
.ForceTopicCreation(true)
.SubscriptionName("myTopic-sub-Exclusive")
.SubscriptionInitialPosition(SharpPulsar.Common.SubscriptionInitialPosition.Earliest));
for (var i = 0; i < 100; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"message '{res}' from topic: {message.TopicName}");
}
}
}
public void ZeroQueueSizeNormalConsumer()
{
string key = "nonZeroQueueSizeNormalConsumer";
// 1. Config
string topicName = "topic-" + key;
string subscriptionName = "my-ex-subscription-" + key;
string messagePredicate = "my-message-" + key + "-";
// 2. Create Producer
var pBuilder = new ProducerConfigBuilder <sbyte[]>()
.Topic(topicName)
.EnableBatching(false);
var producer = _client.NewProducer(pBuilder);
// 3. Create Consumer
var config = new ConsumerConfigBuilder <sbyte[]>()
.Topic(topicName)
.SubscriptionName(subscriptionName)
.ReceiverQueueSize(0);
var consumer = _client.NewConsumer(config);
// 3. producer publish messages
for (int i = 0; i < _totalMessages; i++)
{
var msg = messagePredicate + i;
_output.WriteLine("Producer produced: " + msg);
producer.Send(Encoding.UTF8.GetBytes(msg).ToSBytes());
}
// 4. Receiver receives the message
IMessage <sbyte[]> message;
for (int i = 0; i < _totalMessages; i++)
{
Assert.Equal(0, consumer.NumMessagesInQueue());
message = consumer.Receive();
var r = Encoding.UTF8.GetString(message.Data.ToBytes());
Assert.Equal(r, messagePredicate + i);
Assert.Equal(0, consumer.NumMessagesInQueue());
_output.WriteLine("Consumer received : " + r);
}
}
public void TestUnAckMessageRedeliveryWithReceive()
{
var topic = $"persistent://public/default/async-unack-redelivery-{Guid.NewGuid()}";
var builder = new ConsumerConfigBuilder <byte[]>();
builder.Topic(topic);
builder.SubscriptionName("sub-TestUnAckMessageRedeliveryWithReceive");
builder.AckTimeout(TimeSpan.FromMilliseconds(8000));
builder.ForceTopicCreation(true);
builder.AcknowledgmentGroupTime(0);
builder.SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
var producer = _client.NewProducer(pBuilder);
const int messageCount = 10;
for (var i = 0; i < messageCount; i++)
{
var receipt = producer.Send(Encoding.UTF8.GetBytes("my-message-" + i));
_output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions {
WriteIndented = true
}));
}
var messageReceived = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; ++i)
{
var m = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(m.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.NotNull(receivedMessage);
messageReceived++;
}
Assert.Equal(10, messageReceived);
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < messageCount; i++)
{
var m = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(m.Data);
_output.WriteLine($"Received message: [{receivedMessage}]");
Assert.NotNull(receivedMessage);
messageReceived++;
}
Assert.Equal(20, messageReceived);
producer.Close();
consumer.Close();
}
private static void Transaction(PulsarClient pulsarClient)
{
var txn = (Transaction)pulsarClient.NewTransaction().WithTransactionTimeout(TimeSpan.FromMinutes(5)).Build();
var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>()
.SendTimeout(0)
.Topic(myTopic));
var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(myTopic)
.ForceTopicCreation(true)
.SubscriptionName("myTopic-sub"));
for (var i = 0; i < 10; i++)
{
var text = $"tuts-{i}";
var data = Encoding.UTF8.GetBytes(text);
producer.NewMessage(txn).Value(data).Send();
Console.WriteLine($"produced: {text}");
}
var pool = ArrayPool <byte> .Shared;
//Should not consume messages as the transaction is not committed yet
for (var i = 0; i < 10; ++i)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"[1] message '{res}' from topic: {message.TopicName}");
}
}
txn.Commit();
for (var i = 0; i < 10; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"[2] message '{res}' from topic: {message.TopicName}");
}
}
}
public void TestDeliverAfter()
{
var numMessages = 5;
var consumer = _client.NewConsumer(ISchema <string> .String, new ConsumerConfigBuilder <string>()
.Topic(_topic)
.SubscriptionName($"delayed-sub-{Guid.NewGuid()}")
//deliverat works with shared subscription
.SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared)
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
var producer = _client.NewProducer(ISchema <string> .String, new ProducerConfigBuilder <string>()
.Topic(_topic));
// delay 5 seconds using DeliverAfter
for (var i = 0; i < numMessages; i++)
{
producer.NewMessage().Value("DeliverAfter message " + i).DeliverAfter(TimeSpan.FromMilliseconds(5000)).Send();
}
producer.Flush();
var numReceived = 0;
while (numMessages <= 0 || numReceived < numMessages)
{
var msg = consumer.Receive();
if (msg == null)
{
Thread.Sleep(TimeSpan.FromSeconds(1));
continue;
}
var dt = (DateTimeHelper.CurrentUnixTimeMillis() - msg.PublishTime) / 1000;
_output.WriteLine("Consumer Received message : " + msg.Value + "; Difference between publish time and receive time = " + dt + " seconds");
consumer.Acknowledge(msg);
++numReceived;
}
_output.WriteLine("Successfully received " + numReceived + " messages");
producer.Close();
consumer.Close();
}
public void TestLargeMessage()
{
//this.conf.MaxMessageSize = 5;
const int totalMessages = 3;
var topicName = $"persistent://public/default/my-topic1-{DateTimeHelper.CurrentUnixTimeMillis()}";
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(topicName)
.SubscriptionName("my-subscriber-name")
.AckTimeout(TimeSpan.FromMilliseconds(20000))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topicName)
.EnableChunking(true)
.MaxMessageSize(5);
var producer = _client.NewProducer(pBuilder);
IList <string> publishedMessages = new List <string>();
for (var i = 1; i < totalMessages; i++)
{
var message = CreateMessagePayload(i * 10);
publishedMessages.Add(message);
producer.Send(Encoding.UTF8.GetBytes(message));
}
IMessage <byte[]> msg = null;
ISet <string> messageSet = new HashSet <string>();
IList <IMessage <byte[]> > msgIds = new List <IMessage <byte[]> >();
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < totalMessages - 1; i++)
{
msg = consumer.Receive();
var receivedMessage = Encoding.UTF8.GetString(msg.Data);
_output.WriteLine($"[{i}] - Published [{publishedMessages[i]}] Received message: [{receivedMessage}]");
var expectedMessage = publishedMessages[i];
TestMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
msgIds.Add(msg);
}
foreach (var msgId in msgIds)
{
consumer.Acknowledge(msgId);
}
producer.Close();
consumer.Close();
}
public void Produce_Consumer_Allocation()
{
var topic = $"memory-allocation-{Guid.NewGuid()}";
var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName($"myTopic-sub-{Guid.NewGuid()}")
.IsAckReceiptEnabled(true)
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topic));
for (var i = 0; i < 500; i++)
{
var data = Encoding.UTF8.GetBytes($"memory-allocation-{i}");
producer.NewMessage().Value(data).Send();
}
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < 500; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
_output.WriteLine($"message '{res}' from topic: {message.Topic}");
}
}
dotMemory.Check(memory =>
{
var result = memory
.GetObjects(where => @where.Namespace.Like("SharpPulsar.*"));
_output.WriteLine(
ConsoleTable.From(
result
.GroupByType()
.Select(t => new
{
t.Type.Name,
t.ObjectsCount,
t.SizeInBytes
})
.OrderByDescending(x => x.SizeInBytes)
).ToMarkDownString());
Assert.True(result.ObjectsCount > 0);
});
}
private Consumer <byte[]> CreateConsumer(string topic, string consumerSub, KeySharedPolicy keySharedPolicy = null)
{
var builder = new ConsumerConfigBuilder <byte[]>();
builder.Topic(topic);
builder.SubscriptionName(consumerSub);
builder.AckTimeout(TimeSpan.FromSeconds(10));
builder.ForceTopicCreation(true);
if (keySharedPolicy != null)
{
builder.KeySharedPolicy(keySharedPolicy);
}
builder.SubscriptionType(CommandSubscribe.SubType.KeyShared);
return(_client.NewConsumer(builder));
}
static async Task Read()
{
var consumer = await client.NewConsumer(Schema.JSON <Test>())
.Topic(topicName)
.SubscriptionName("test")
.SubscribeAsync();
while (!consumer.HasReachedEndOfTopic && running)
{
var message = await consumer.ReceiveAsync();
Console.WriteLine($"Received: {message.GetValue().Text}");
await consumer.AcknowledgeAsync(message.MessageId);
}
}
public void TestAddAndRemove()
{
var builder = new ConsumerConfigBuilder <byte[]>();
builder.Topic("TestAckTracker");
builder.SubscriptionName("TestAckTracker-sub");
var consumer = _client.NewConsumer(builder);
var unack = _system.ActorOf(UnAckedChunckedMessageIdSequenceMap.Prop());
var tracker = _client.ActorSystem.ActorOf(UnAckedMessageTracker.Prop(1000000, 100000, consumer.ConsumerActor, unack));
var empty = tracker.Ask <bool>(Empty.Instance).GetAwaiter().GetResult();
Assert.True(empty);
var size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(0, size);
var mid = new MessageId(1L, 1L, -1);
var added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult();
Assert.True(added);
added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult();
Assert.False(added);
size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(1, size);
tracker.Tell(Clear.Instance);
added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult();
Assert.True(added);
size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(1, size);
var removed = tracker.Ask <bool>(new Remove(mid)).GetAwaiter().GetResult();
Assert.True(removed);
empty = tracker.Ask <bool>(Empty.Instance).GetAwaiter().GetResult();
Assert.True(empty);
size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult();
Assert.Equal(0, size);
}
private static void MultiConsumer(PulsarClient client)
{
var messageCount = 5;
var first = $"one-topic-{Guid.NewGuid()}";
var second = $"two-topic-{Guid.NewGuid()}";
var third = $"three-topic-{Guid.NewGuid()}";
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(first, second, third)
.ForceTopicCreation(true)
.SubscriptionName("multi-topic-sub");
var consumer = client.NewConsumer(builder);
PublishMessages(first, messageCount, "hello Toba", client);
PublishMessages(third, messageCount, "hello Toba", client);
PublishMessages(second, messageCount, "hello Toba", client);
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
if (message != null)
{
consumer.Acknowledge(message);
Console.WriteLine($"message from topic: {message.TopicName}");
}
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
if (message != null)
{
consumer.Acknowledge(message);
Console.WriteLine($"message from topic: {message.TopicName}");
}
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive(); if (message != null)
{
consumer.Acknowledge(message);
Console.WriteLine($"message from topic: {message.TopicName}");
}
}
}
public void Setup()
{
var clientConfig = new PulsarClientConfigBuilder()
.ServiceUrl("pulsar://localhost:6650");
_pulsarSystem = PulsarSystem.GetInstance(clientConfig);
_client = _pulsarSystem.NewClient();
_consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(_benchTopic)
.ForceTopicCreation(true)
.SubscriptionName($"bench-sub-{Guid.NewGuid()}")
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
_producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(_benchTopic));
}
public void TestEncrptedProduceConsume()
{
var messageCount = 10;
var topic = $"encrypted-messages-{Guid.NewGuid()}";
var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.ForceTopicCreation(true)
.CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem"))
.SubscriptionName("encrypted-sub")
.SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest));
var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem"))
.AddEncryptionKey("Ebere"));
for (var i = 0; i < messageCount; i++)
{
producer.Send(Encoding.UTF8.GetBytes($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]"));
}
var receivedCount = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; i++)
{
var message = consumer.Receive();
if (message != null)
{
var decrypted = Encoding.UTF8.GetString(message.Data);
_output.WriteLine(decrypted);
Assert.Equal($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]", decrypted);
consumer.Acknowledge(message);
receivedCount++;
}
}
Assert.True(receivedCount > 6);
producer.Close();
consumer.Close();
}
public void TestMultiTopicConsumer()
{
var messageCount = 5;
var first = $"one-topic-{Guid.NewGuid()}";
var second = $"two-topic-{Guid.NewGuid()}";
var third = $"three-topic-{Guid.NewGuid()}";
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(first, second, third)
.ForceTopicCreation(true)
.SubscriptionName("multi-topic-sub");
var consumer = _client.NewConsumer(builder);
PublishMessages(first, messageCount, "hello Toba");
PublishMessages(third, messageCount, "hello Toba");
PublishMessages(second, messageCount, "hello Toba");
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
_output.WriteLine($"message from topic: {message.Topic}");
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
Assert.NotNull(message);
_output.WriteLine($"message from topic: {message.Topic}");
}
for (var i = 0; i < messageCount; i++)
{
var message = (TopicMessage <byte[]>)consumer.Receive();
Assert.NotNull(message);
_output.WriteLine($"message from topic: {message.Topic}");
}
consumer.Close();
}
private static void ProduceConsumer(PulsarClient pulsarClient)
{
var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.Topic(myTopic)
.ForceTopicCreation(true)
.SubscriptionName($"sub-{Guid.NewGuid()}")
.IsAckReceiptEnabled(true));
var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(myTopic));
for (var i = 0; i < 1000; i++)
{
var data = Encoding.UTF8.GetBytes($"tuts-{i}");
var id = producer.NewMessage().Value(data).Send();
Console.WriteLine($"Message Id({id.LedgerId}:{id.EntryId})");
}
var pool = ArrayPool <byte> .Shared;
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < 1000; i++)
{
var message = (Message <byte[]>)consumer.Receive();
if (message != null)
{
var payload = pool.Rent((int)message.Data.Length);
Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length);
consumer.Acknowledge(message);
var res = Encoding.UTF8.GetString(message.Data);
Console.WriteLine($"message '{res}' from topic: {message.TopicName}");
}
}
}
public void TestAckTracker()
{
var builder = new ConsumerConfigBuilder <byte[]>();
builder.AcknowledgmentGroupTime(10000);
builder.Topic($"TestAckTracker-{Guid.NewGuid()}");
builder.SubscriptionName($"TestAckTracker-sub-{Guid.NewGuid()}");
var conf = builder.ConsumerConfigurationData;
var consumer = _client.NewConsumer(builder);
var unack = _system.ActorOf(UnAckedChunckedMessageIdSequenceMap.Prop());
var tracker = _client.ActorSystem.ActorOf(PersistentAcknowledgmentsGroupingTracker <byte[]> .Prop(unack, consumer.ConsumerActor, consumer.ConsumerActor /*dummy*/, 1, consumer.ConsumerActor, conf));
var msg1 = new MessageId(5, 1, 0);
var msg2 = new MessageId(5, 2, 0);
var msg3 = new MessageId(5, 3, 0);
var msg4 = new MessageId(5, 4, 0);
var msg5 = new MessageId(5, 5, 0);
var msg6 = new MessageId(5, 6, 0);
var isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg1, CommandAck.AckType.Individual, new Dictionary <string, long>()));
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg5, CommandAck.AckType.Cumulative, new Dictionary <string, long>()));
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
// Flush while disconnected. the internal tracking will not change
tracker.Tell(FlushPending.Instance);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.Tell(new AddAcknowledgment(msg6, CommandAck.AckType.Individual, new Dictionary <string, long>()));
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
tracker.Tell(FlushPending.Instance);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult();
Assert.True(isDuplicate);
isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult();
Assert.False(isDuplicate);
tracker.GracefulStop(TimeSpan.FromSeconds(1));
}
public void TestGenericTopic()
{
var schema = AvroSchema <ComplexGenericData> .Of(typeof(ComplexGenericData));
var genericSchema = GenericAvroSchema.Of(schema.SchemaInfo);
_output.WriteLine(schema.SchemaInfo.SchemaDefinition);
var pBuilder = new ProducerConfigBuilder <IGenericRecord>()
.Topic(_topic);
var producer = _client.NewProducer(genericSchema, pBuilder);
const int messageCount = 10;
for (var i = 0; i < messageCount; i++)
{
var dataForWriter = new GenericRecord((Avro.RecordSchema)genericSchema.AvroSchema);
dataForWriter.Add("Feature", "Education");
dataForWriter.Add("StringData", new Dictionary <string, string> {
{ "Index", i.ToString() }, { "FirstName", "Ebere" }, { "LastName", "Abanonu" }
});
dataForWriter.Add("ComplexData", ToBytes(new ComplexData {
ProductId = i, Point = i * 2, Sales = i * 2 * 5
}));
var record = new GenericAvroRecord(null, genericSchema.AvroSchema, genericSchema.Fields, dataForWriter);
var receipt = producer.Send(record);
_output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions {
WriteIndented = true
}));
}
var messageReceived = 0;
var builder = new ConsumerConfigBuilder <IGenericRecord>()
.Topic(_topic)
.ForceTopicCreation(true)
.SubscriptionName($"generic_sub");
var consumer = _client.NewConsumer(ISchema <object> .AutoConsume(), builder);
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCount; ++i)
{
var m = consumer.Receive();
Assert.NotNull(m);
var receivedMessage = m.Value;
var feature = receivedMessage.GetField("Feature").ToString();
var strinData = (Dictionary <string, object>)receivedMessage.GetField("StringData");
var complexData = FromBytes <ComplexData>((byte[])receivedMessage.GetField("ComplexData"));
_output.WriteLine(feature);
_output.WriteLine(JsonSerializer.Serialize(strinData, new JsonSerializerOptions {
WriteIndented = true
}));
_output.WriteLine(JsonSerializer.Serialize(complexData, new JsonSerializerOptions {
WriteIndented = true
}));
messageReceived++;
consumer.Acknowledge(m);
}
Assert.Equal(10, messageReceived);
producer.Close();
consumer.Close();
}
public void TxnAckTestBatchedFailoverSub()
{
var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(normalTopic)
.SubscriptionName($"test-{Guid.NewGuid()}")
.ForceTopicCreation(true)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(5000)
.SubscriptionType(SubType.Failover);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(normalTopic)
.EnableBatching(true)
.BatchingMaxMessages(100);
var producer = _client.NewProducer(producerBuilder);
for (var retryCnt = 0; retryCnt < 1; retryCnt++)
{
var txn = Txn;
//Thread.Sleep(TimeSpan.FromSeconds(30));
var messageCnt = 100;
// produce normal messages
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage().Value("hello".GetBytes()).Send();
}
// consume and ack messages with txn
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
var msg = consumer.Receive();
Assert.NotNull(msg);
_output.WriteLine($"receive msgId: {msg.MessageId}, count : {i}");
consumer.Acknowledge(msg.MessageId, txn);
}
// the messages are pending ack state and can't be received
var message = consumer.Receive();
Assert.Null(message);
// 1) txn abort
txn.Abort();
// after transaction abort, the messages could be received
var commitTxn = Txn;
//Thread.Sleep(TimeSpan.FromSeconds(5));
Thread.Sleep(TimeSpan.FromSeconds(30));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message.MessageId, commitTxn);
_output.WriteLine($"receive msgId: {message.MessageId}, count: {i}");
}
// 2) ack committed by a new txn
commitTxn.Commit();
// after transaction commit, the messages can't be received
message = consumer.Receive();
Assert.Null(message);
}
}
public void TxnCumulativeAckTest()
{
var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(normalTopic)
.SubscriptionName($"test-{Guid.NewGuid()}")
.ForceTopicCreation(true)
//.SubscriptionType(SubType.Failover)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(3000)
.AckTimeout(TimeSpan.FromMilliseconds(10000));
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(normalTopic);
var producer = _client.NewProducer(producerBuilder);
for (var retryCnt = 0; retryCnt < 2; retryCnt++)
{
var abortTxn = Txn;
var messageCnt = 50;
// produce normal messages
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage().Value(Encoding.UTF8.GetBytes("Hello")).Send();
}
IMessage <byte[]> message = null;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
if (i % 3 == 0)
{
consumer.AcknowledgeCumulative(message.MessageId, abortTxn);
}
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
// the messages are pending ack state and can't be received
message = consumer.Receive();
Assert.Null(message);
abortTxn.Abort();
var commitTxn = Txn;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.AcknowledgeCumulative(message.MessageId, commitTxn);
_output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}");
}
commitTxn.Commit();
Thread.Sleep(TimeSpan.FromSeconds(5));
message = consumer.Receive();
Assert.Null(message);
}
}
private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout)
{
_output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}");
var topic = "testNegativeAcks-" + DateTime.Now.Ticks;
var builder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName($"sub1-{Guid.NewGuid()}")
.AckTimeout(TimeSpan.FromMilliseconds(ackTimeout))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0)
.NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis))
.SubscriptionType(subscriptionType);
var consumer = _client.NewConsumer(builder);
var pBuilder = new ProducerConfigBuilder <byte[]>();
pBuilder.Topic(topic);
if (batching)
{
pBuilder.EnableBatching(batching);
pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis));
pBuilder.BatchingMaxMessages(10);
}
var producer = _client.NewProducer(pBuilder);
ISet <string> sentMessages = new HashSet <string>();
const int n = 10;
for (var i = 0; i < n; i++)
{
var value = "test-" + i;
producer.Send(Encoding.UTF8.GetBytes(value));
sentMessages.Add(value);
}
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
consumer.NegativeAcknowledge(msg);
_output.WriteLine(ms);
}
}
ISet <string> receivedMessages = new HashSet <string>();
Thread.Sleep(TimeSpan.FromSeconds(10));
// All the messages should be received again
for (var i = 0; i < n; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var ms = Encoding.UTF8.GetString(msg.Data);
_output.WriteLine(ms);
receivedMessages.Add(ms);
consumer.Acknowledge(msg);
}
}
Assert.Equal(sentMessages, receivedMessages);
var nu = consumer.Receive();
// There should be no more messages
Assert.Null(nu);
producer.Close();
consumer.Close();
}
public void TxnMessageAckTest()
{
var topic = $"{_topicMessageAckTest}-{Guid.NewGuid()}";
var subName = $"test-{Guid.NewGuid()}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.SubscriptionName(subName)
.ForceTopicCreation(true)
.EnableBatchIndexAcknowledgment(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.EnableBatching(false)
.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var txn = Txn;
var messageCnt = 10;
for (var i = 0; i < messageCnt; i++)
{
producer.NewMessage(txn).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
_output.WriteLine("produce transaction messages finished");
// Can't receive transaction messages before commit.
var message = consumer.Receive();
Assert.Null(message);
_output.WriteLine("transaction messages can't be received before transaction committed");
txn.Commit();
var ackedMessageCount = 0;
var receiveCnt = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
receiveCnt++;
if (i % 2 == 0)
{
consumer.Acknowledge(message);
ackedMessageCount++;
}
}
Assert.Equal(messageCnt, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
consumer.RedeliverUnacknowledgedMessages();
Thread.Sleep(TimeSpan.FromSeconds(10));
receiveCnt = 0;
for (var i = 0; i < messageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
receiveCnt++;
}
Assert.Equal(messageCnt - ackedMessageCount, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
_output.WriteLine($"receive transaction messages count: {receiveCnt}");
}
public virtual void TestBinaryProtoToGetTopicsOfNamespacePersistent()
{
var key = Guid.NewGuid().ToString();
var subscriptionName = "regex-subscription";
var topicName1 = "persistent://public/default/reg-topic-1-" + key;
var topicName2 = "persistent://public/default/reg-topic-2-" + key;
var topicName3 = "persistent://public/default/reg-topic-3-" + key;
var topicName4 = "non-persistent://public/default/reg-topic-4-" + key;
var pattern = new Regex("public/default/reg-topic*");
// 2. create producer
var messagePredicate = "my-message-" + key + "-";
var producer1 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName1));
var producer2 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName2));
var producer3 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName3));
var producer4 = _client.NewProducer(new ProducerConfigBuilder <byte[]>()
.Topic(topicName4));
// 5. produce data
for (var i = 0; i < 10; i++)
{
producer1.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer1-" + i));
producer2.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer2-" + i));
producer3.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer3-" + i));
producer4.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer4-" + i));
}
// 6. should receive all the message
var messageSet = 0;
var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>()
.TopicsPattern(pattern)
.PatternAutoDiscoveryPeriod(2)
.ForceTopicCreation(true)
.SubscriptionName(subscriptionName)
.SubscriptionType(SubType.Shared)
.AckTimeout(TimeSpan.FromMilliseconds(60000)));
Thread.Sleep(TimeSpan.FromSeconds(10));
var message = consumer.Receive();
if (message == null)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
message = consumer.Receive();
}
while (message != null)
{
var m = (TopicMessage <byte[]>)message;
messageSet++;
consumer.Acknowledge(m);
_output.WriteLine($"Consumer acknowledged : {Encoding.UTF8.GetString(message.Data)} from topic: {m.Topic}");
message = consumer.Receive();
}
consumer.Unsubscribe();
consumer.Close();
producer1.Close();
producer2.Close();
producer3.Close();
producer4.Close();
Assert.True(messageSet > 0);
}
private void PlainAvroProducer(string topic)
{
var jsonSchem = AvroSchema <JournalEntry> .Of(typeof(JournalEntry));
var builder = new ConsumerConfigBuilder <JournalEntry>()
.Topic(topic)
.SubscriptionName($"my-subscriber-name-{DateTimeHelper.CurrentUnixTimeMillis()}")
.AckTimeout(TimeSpan.FromMilliseconds(20000))
.ForceTopicCreation(true)
.AcknowledgmentGroupTime(0);
var consumer = _client.NewConsumer(jsonSchem, builder);
var producerConfig = new ProducerConfigBuilder <JournalEntry>()
.ProducerName(topic.Split("/").Last())
.Topic(topic)
.Schema(jsonSchem)
.SendTimeout(10000);
var producer = _client.NewProducer(jsonSchem, producerConfig);
for (var i = 0; i < 10; i++)
{
var student = new Students
{
Name = $"[{i}] Ebere: {DateTimeOffset.Now.ToUnixTimeMilliseconds()} - presto-ed {DateTime.Now.ToString(CultureInfo.InvariantCulture)}",
Age = 202 + i,
School = "Akka-Pulsar university"
};
var journal = new JournalEntry
{
Id = $"[{i}]Ebere: {DateTimeOffset.Now.ToUnixTimeMilliseconds()}",
PersistenceId = "sampleActor",
IsDeleted = false,
Ordering = 0,
Payload = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(student)),
SequenceNr = 0,
Tags = "root"
};
var metadata = new Dictionary <string, string>
{
["Key"] = "Single",
["Properties"] = JsonSerializer.Serialize(new Dictionary <string, string> {
{ "Tick", DateTime.Now.Ticks.ToString() }
}, new JsonSerializerOptions {
WriteIndented = true
})
};
var id = producer.NewMessage().Properties(metadata).Value(journal).Send();
}
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < 10; i++)
{
var msg = consumer.Receive();
if (msg != null)
{
var receivedMessage = msg.Value;
_output.WriteLine(JsonSerializer.Serialize(receivedMessage, new JsonSerializerOptions {
WriteIndented = true
}));
}
}
}
public void ProduceCommitTest()
{
var txn1 = Txn;
var txn2 = Txn;
var topic = $"{_topicOutput}";
var consumerBuilder = new ConsumerConfigBuilder <byte[]>()
.Topic(topic)
.ForceTopicCreation(true)
.SubscriptionName($"test-{Guid.NewGuid()}");
var consumer = _client.NewConsumer(consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <byte[]>()
.Topic(topic)
.SendTimeout(0);
var producer = _client.NewProducer(producerBuilder);
var txnMessageCnt = 0;
var messageCnt = 10;
for (var i = 0; i < messageCnt; i++)
{
if (i % 5 == 0)
{
producer.NewMessage(txn1).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
else
{
producer.NewMessage(txn2).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send();
}
txnMessageCnt++;
}
// Can't receive transaction messages before commit.
var message = consumer.Receive();
Assert.Null(message);
txn1.Commit();
_output.WriteLine($"Committed 1");
txn2.Commit();
_output.WriteLine($"Committed 1");
// txn1 messages could be received after txn1 committed
var receiveCnt = 0;
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < txnMessageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
_output.WriteLine(Encoding.UTF8.GetString(message.Value));
receiveCnt++;
}
for (var i = 0; i < txnMessageCnt; i++)
{
message = consumer.Receive();
Assert.NotNull(message);
receiveCnt++;
}
Assert.Equal(txnMessageCnt, receiveCnt);
message = consumer.Receive();
Assert.Null(message);
_output.WriteLine($"message commit test enableBatch {true}");
}