public void ProduceAndConsume() { var record1 = AvroSchema <SimpleRecord> .Of(typeof(SimpleRecord)); var consumerBuilder = new ConsumerConfigBuilder <SimpleRecord>() .Topic(_topic) .ConsumerName("avroUpgradeSchema1") .SubscriptionName("test-sub"); var consumer = _client.NewConsumer(record1, consumerBuilder); var producerBuilder = new ProducerConfigBuilder <SimpleRecord>() .Topic(_topic) .ProducerName("avroUpgradeSchema1"); var producer = _client.NewProducer(record1, producerBuilder); producer.NewMessage() .Value(new SimpleRecord { Name = "Ebere Abanonu", Age = int.MaxValue }) .Send(); Thread.Sleep(TimeSpan.FromSeconds(10)); var message = consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message); consumer.Unsubscribe(); var record2 = AvroSchema <SimpleRecord2> .Of(typeof(SimpleRecord2)); var consumerBuilder2 = new ConsumerConfigBuilder <SimpleRecord2>() .Topic(_topic) .ConsumerName("avroUpgradeSchema2") .SubscriptionName("test-sub"); var consumer1 = _client.NewConsumer(record2, consumerBuilder2); var producerBuilder2 = new ProducerConfigBuilder <SimpleRecord2>() .Topic(_topic) .ProducerName("avroUpgradeSchema2"); var producer2 = _client.NewProducer(record2, producerBuilder2); producer2.NewMessage() .Value(new SimpleRecord2 { Name = "Ebere", Age = int.MaxValue, Surname = "Abanonu" }) .Send(); Thread.Sleep(TimeSpan.FromSeconds(10)); var msg = consumer1.Receive(); Assert.NotNull(msg); consumer1.Acknowledge(msg); consumer1.Unsubscribe(); producer.Close(); producer2.Close(); consumer.Close(); consumer1.Close(); }
public virtual void ConsumerInstantiation() { var consumer = new ConsumerConfigBuilder <string>(); consumer.Topic("ConsumerInstantiation"); consumer.SubscriptionName($"test-sub-{Guid.NewGuid()}"); var stringConsumerBuilder = _client.NewConsumer(new StringSchema(), consumer); Assert.NotNull(stringConsumerBuilder); stringConsumerBuilder.Close(); }
private static void ExclusiveProduceConsumer(PulsarClient pulsarClient) { var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>() .AccessMode(SharpPulsar.Common.ProducerAccessMode.Exclusive) .Topic(myTopic));; for (var i = 0; i < 100; i++) { var data = Encoding.UTF8.GetBytes($"tuts-{i}"); producer.NewMessage().Value(data).Send(); } var pool = ArrayPool <byte> .Shared; var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(myTopic) .ForceTopicCreation(true) .SubscriptionName("myTopic-sub-Exclusive") .SubscriptionInitialPosition(SharpPulsar.Common.SubscriptionInitialPosition.Earliest)); for (var i = 0; i < 100; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"message '{res}' from topic: {message.TopicName}"); } } }
public void ZeroQueueSizeNormalConsumer() { string key = "nonZeroQueueSizeNormalConsumer"; // 1. Config string topicName = "topic-" + key; string subscriptionName = "my-ex-subscription-" + key; string messagePredicate = "my-message-" + key + "-"; // 2. Create Producer var pBuilder = new ProducerConfigBuilder <sbyte[]>() .Topic(topicName) .EnableBatching(false); var producer = _client.NewProducer(pBuilder); // 3. Create Consumer var config = new ConsumerConfigBuilder <sbyte[]>() .Topic(topicName) .SubscriptionName(subscriptionName) .ReceiverQueueSize(0); var consumer = _client.NewConsumer(config); // 3. producer publish messages for (int i = 0; i < _totalMessages; i++) { var msg = messagePredicate + i; _output.WriteLine("Producer produced: " + msg); producer.Send(Encoding.UTF8.GetBytes(msg).ToSBytes()); } // 4. Receiver receives the message IMessage <sbyte[]> message; for (int i = 0; i < _totalMessages; i++) { Assert.Equal(0, consumer.NumMessagesInQueue()); message = consumer.Receive(); var r = Encoding.UTF8.GetString(message.Data.ToBytes()); Assert.Equal(r, messagePredicate + i); Assert.Equal(0, consumer.NumMessagesInQueue()); _output.WriteLine("Consumer received : " + r); } }
public void TestUnAckMessageRedeliveryWithReceive() { var topic = $"persistent://public/default/async-unack-redelivery-{Guid.NewGuid()}"; var builder = new ConsumerConfigBuilder <byte[]>(); builder.Topic(topic); builder.SubscriptionName("sub-TestUnAckMessageRedeliveryWithReceive"); builder.AckTimeout(TimeSpan.FromMilliseconds(8000)); builder.ForceTopicCreation(true); builder.AcknowledgmentGroupTime(0); builder.SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared); var consumer = _client.NewConsumer(builder); var pBuilder = new ProducerConfigBuilder <byte[]>(); pBuilder.Topic(topic); var producer = _client.NewProducer(pBuilder); const int messageCount = 10; for (var i = 0; i < messageCount; i++) { var receipt = producer.Send(Encoding.UTF8.GetBytes("my-message-" + i)); _output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions { WriteIndented = true })); } var messageReceived = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCount; ++i) { var m = consumer.Receive(); var receivedMessage = Encoding.UTF8.GetString(m.Data); _output.WriteLine($"Received message: [{receivedMessage}]"); Assert.NotNull(receivedMessage); messageReceived++; } Assert.Equal(10, messageReceived); Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < messageCount; i++) { var m = consumer.Receive(); var receivedMessage = Encoding.UTF8.GetString(m.Data); _output.WriteLine($"Received message: [{receivedMessage}]"); Assert.NotNull(receivedMessage); messageReceived++; } Assert.Equal(20, messageReceived); producer.Close(); consumer.Close(); }
private static void Transaction(PulsarClient pulsarClient) { var txn = (Transaction)pulsarClient.NewTransaction().WithTransactionTimeout(TimeSpan.FromMinutes(5)).Build(); var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>() .SendTimeout(0) .Topic(myTopic)); var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(myTopic) .ForceTopicCreation(true) .SubscriptionName("myTopic-sub")); for (var i = 0; i < 10; i++) { var text = $"tuts-{i}"; var data = Encoding.UTF8.GetBytes(text); producer.NewMessage(txn).Value(data).Send(); Console.WriteLine($"produced: {text}"); } var pool = ArrayPool <byte> .Shared; //Should not consume messages as the transaction is not committed yet for (var i = 0; i < 10; ++i) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"[1] message '{res}' from topic: {message.TopicName}"); } } txn.Commit(); for (var i = 0; i < 10; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"[2] message '{res}' from topic: {message.TopicName}"); } } }
public void TestDeliverAfter() { var numMessages = 5; var consumer = _client.NewConsumer(ISchema <string> .String, new ConsumerConfigBuilder <string>() .Topic(_topic) .SubscriptionName($"delayed-sub-{Guid.NewGuid()}") //deliverat works with shared subscription .SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared) .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); var producer = _client.NewProducer(ISchema <string> .String, new ProducerConfigBuilder <string>() .Topic(_topic)); // delay 5 seconds using DeliverAfter for (var i = 0; i < numMessages; i++) { producer.NewMessage().Value("DeliverAfter message " + i).DeliverAfter(TimeSpan.FromMilliseconds(5000)).Send(); } producer.Flush(); var numReceived = 0; while (numMessages <= 0 || numReceived < numMessages) { var msg = consumer.Receive(); if (msg == null) { Thread.Sleep(TimeSpan.FromSeconds(1)); continue; } var dt = (DateTimeHelper.CurrentUnixTimeMillis() - msg.PublishTime) / 1000; _output.WriteLine("Consumer Received message : " + msg.Value + "; Difference between publish time and receive time = " + dt + " seconds"); consumer.Acknowledge(msg); ++numReceived; } _output.WriteLine("Successfully received " + numReceived + " messages"); producer.Close(); consumer.Close(); }
public void TestLargeMessage() { //this.conf.MaxMessageSize = 5; const int totalMessages = 3; var topicName = $"persistent://public/default/my-topic1-{DateTimeHelper.CurrentUnixTimeMillis()}"; var builder = new ConsumerConfigBuilder <byte[]>() .Topic(topicName) .SubscriptionName("my-subscriber-name") .AckTimeout(TimeSpan.FromMilliseconds(20000)) .ForceTopicCreation(true) .AcknowledgmentGroupTime(0); var consumer = _client.NewConsumer(builder); var pBuilder = new ProducerConfigBuilder <byte[]>() .Topic(topicName) .EnableChunking(true) .MaxMessageSize(5); var producer = _client.NewProducer(pBuilder); IList <string> publishedMessages = new List <string>(); for (var i = 1; i < totalMessages; i++) { var message = CreateMessagePayload(i * 10); publishedMessages.Add(message); producer.Send(Encoding.UTF8.GetBytes(message)); } IMessage <byte[]> msg = null; ISet <string> messageSet = new HashSet <string>(); IList <IMessage <byte[]> > msgIds = new List <IMessage <byte[]> >(); Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < totalMessages - 1; i++) { msg = consumer.Receive(); var receivedMessage = Encoding.UTF8.GetString(msg.Data); _output.WriteLine($"[{i}] - Published [{publishedMessages[i]}] Received message: [{receivedMessage}]"); var expectedMessage = publishedMessages[i]; TestMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); msgIds.Add(msg); } foreach (var msgId in msgIds) { consumer.Acknowledge(msgId); } producer.Close(); consumer.Close(); }
public void Produce_Consumer_Allocation() { var topic = $"memory-allocation-{Guid.NewGuid()}"; var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(topic) .SubscriptionName($"myTopic-sub-{Guid.NewGuid()}") .IsAckReceiptEnabled(true) .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topic)); for (var i = 0; i < 500; i++) { var data = Encoding.UTF8.GetBytes($"memory-allocation-{i}"); producer.NewMessage().Value(data).Send(); } Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < 500; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); _output.WriteLine($"message '{res}' from topic: {message.Topic}"); } } dotMemory.Check(memory => { var result = memory .GetObjects(where => @where.Namespace.Like("SharpPulsar.*")); _output.WriteLine( ConsoleTable.From( result .GroupByType() .Select(t => new { t.Type.Name, t.ObjectsCount, t.SizeInBytes }) .OrderByDescending(x => x.SizeInBytes) ).ToMarkDownString()); Assert.True(result.ObjectsCount > 0); }); }
private Consumer <byte[]> CreateConsumer(string topic, string consumerSub, KeySharedPolicy keySharedPolicy = null) { var builder = new ConsumerConfigBuilder <byte[]>(); builder.Topic(topic); builder.SubscriptionName(consumerSub); builder.AckTimeout(TimeSpan.FromSeconds(10)); builder.ForceTopicCreation(true); if (keySharedPolicy != null) { builder.KeySharedPolicy(keySharedPolicy); } builder.SubscriptionType(CommandSubscribe.SubType.KeyShared); return(_client.NewConsumer(builder)); }
static async Task Read() { var consumer = await client.NewConsumer(Schema.JSON <Test>()) .Topic(topicName) .SubscriptionName("test") .SubscribeAsync(); while (!consumer.HasReachedEndOfTopic && running) { var message = await consumer.ReceiveAsync(); Console.WriteLine($"Received: {message.GetValue().Text}"); await consumer.AcknowledgeAsync(message.MessageId); } }
public void TestAddAndRemove() { var builder = new ConsumerConfigBuilder <byte[]>(); builder.Topic("TestAckTracker"); builder.SubscriptionName("TestAckTracker-sub"); var consumer = _client.NewConsumer(builder); var unack = _system.ActorOf(UnAckedChunckedMessageIdSequenceMap.Prop()); var tracker = _client.ActorSystem.ActorOf(UnAckedMessageTracker.Prop(1000000, 100000, consumer.ConsumerActor, unack)); var empty = tracker.Ask <bool>(Empty.Instance).GetAwaiter().GetResult(); Assert.True(empty); var size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult(); Assert.Equal(0, size); var mid = new MessageId(1L, 1L, -1); var added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult(); Assert.True(added); added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult(); Assert.False(added); size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult(); Assert.Equal(1, size); tracker.Tell(Clear.Instance); added = tracker.Ask <bool>(new Add(mid)).GetAwaiter().GetResult(); Assert.True(added); size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult(); Assert.Equal(1, size); var removed = tracker.Ask <bool>(new Remove(mid)).GetAwaiter().GetResult(); Assert.True(removed); empty = tracker.Ask <bool>(Empty.Instance).GetAwaiter().GetResult(); Assert.True(empty); size = tracker.Ask <long>(Size.Instance).GetAwaiter().GetResult(); Assert.Equal(0, size); }
private static void MultiConsumer(PulsarClient client) { var messageCount = 5; var first = $"one-topic-{Guid.NewGuid()}"; var second = $"two-topic-{Guid.NewGuid()}"; var third = $"three-topic-{Guid.NewGuid()}"; var builder = new ConsumerConfigBuilder <byte[]>() .Topic(first, second, third) .ForceTopicCreation(true) .SubscriptionName("multi-topic-sub"); var consumer = client.NewConsumer(builder); PublishMessages(first, messageCount, "hello Toba", client); PublishMessages(third, messageCount, "hello Toba", client); PublishMessages(second, messageCount, "hello Toba", client); for (var i = 0; i < messageCount; i++) { var message = (TopicMessage <byte[]>)consumer.Receive(); if (message != null) { consumer.Acknowledge(message); Console.WriteLine($"message from topic: {message.TopicName}"); } } for (var i = 0; i < messageCount; i++) { var message = (TopicMessage <byte[]>)consumer.Receive(); if (message != null) { consumer.Acknowledge(message); Console.WriteLine($"message from topic: {message.TopicName}"); } } for (var i = 0; i < messageCount; i++) { var message = (TopicMessage <byte[]>)consumer.Receive(); if (message != null) { consumer.Acknowledge(message); Console.WriteLine($"message from topic: {message.TopicName}"); } } }
public void Setup() { var clientConfig = new PulsarClientConfigBuilder() .ServiceUrl("pulsar://localhost:6650"); _pulsarSystem = PulsarSystem.GetInstance(clientConfig); _client = _pulsarSystem.NewClient(); _consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(_benchTopic) .ForceTopicCreation(true) .SubscriptionName($"bench-sub-{Guid.NewGuid()}") .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); _producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(_benchTopic)); }
public void TestEncrptedProduceConsume() { var messageCount = 10; var topic = $"encrypted-messages-{Guid.NewGuid()}"; var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(topic) .ForceTopicCreation(true) .CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem")) .SubscriptionName("encrypted-sub") .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topic) .CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem")) .AddEncryptionKey("Ebere")); for (var i = 0; i < messageCount; i++) { producer.Send(Encoding.UTF8.GetBytes($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]")); } var receivedCount = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCount; i++) { var message = consumer.Receive(); if (message != null) { var decrypted = Encoding.UTF8.GetString(message.Data); _output.WriteLine(decrypted); Assert.Equal($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]", decrypted); consumer.Acknowledge(message); receivedCount++; } } Assert.True(receivedCount > 6); producer.Close(); consumer.Close(); }
public void TestMultiTopicConsumer() { var messageCount = 5; var first = $"one-topic-{Guid.NewGuid()}"; var second = $"two-topic-{Guid.NewGuid()}"; var third = $"three-topic-{Guid.NewGuid()}"; var builder = new ConsumerConfigBuilder <byte[]>() .Topic(first, second, third) .ForceTopicCreation(true) .SubscriptionName("multi-topic-sub"); var consumer = _client.NewConsumer(builder); PublishMessages(first, messageCount, "hello Toba"); PublishMessages(third, messageCount, "hello Toba"); PublishMessages(second, messageCount, "hello Toba"); for (var i = 0; i < messageCount; i++) { var message = (TopicMessage <byte[]>)consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message); _output.WriteLine($"message from topic: {message.Topic}"); } for (var i = 0; i < messageCount; i++) { var message = (TopicMessage <byte[]>)consumer.Receive(); Assert.NotNull(message); _output.WriteLine($"message from topic: {message.Topic}"); } for (var i = 0; i < messageCount; i++) { var message = (TopicMessage <byte[]>)consumer.Receive(); Assert.NotNull(message); _output.WriteLine($"message from topic: {message.Topic}"); } consumer.Close(); }
private static void ProduceConsumer(PulsarClient pulsarClient) { var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(myTopic) .ForceTopicCreation(true) .SubscriptionName($"sub-{Guid.NewGuid()}") .IsAckReceiptEnabled(true)); var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(myTopic)); for (var i = 0; i < 1000; i++) { var data = Encoding.UTF8.GetBytes($"tuts-{i}"); var id = producer.NewMessage().Value(data).Send(); Console.WriteLine($"Message Id({id.LedgerId}:{id.EntryId})"); } var pool = ArrayPool <byte> .Shared; Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < 1000; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"message '{res}' from topic: {message.TopicName}"); } } }
public void TestAckTracker() { var builder = new ConsumerConfigBuilder <byte[]>(); builder.AcknowledgmentGroupTime(10000); builder.Topic($"TestAckTracker-{Guid.NewGuid()}"); builder.SubscriptionName($"TestAckTracker-sub-{Guid.NewGuid()}"); var conf = builder.ConsumerConfigurationData; var consumer = _client.NewConsumer(builder); var unack = _system.ActorOf(UnAckedChunckedMessageIdSequenceMap.Prop()); var tracker = _client.ActorSystem.ActorOf(PersistentAcknowledgmentsGroupingTracker <byte[]> .Prop(unack, consumer.ConsumerActor, consumer.ConsumerActor /*dummy*/, 1, consumer.ConsumerActor, conf)); var msg1 = new MessageId(5, 1, 0); var msg2 = new MessageId(5, 2, 0); var msg3 = new MessageId(5, 3, 0); var msg4 = new MessageId(5, 4, 0); var msg5 = new MessageId(5, 5, 0); var msg6 = new MessageId(5, 6, 0); var isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult(); Assert.False(isDuplicate); tracker.Tell(new AddAcknowledgment(msg1, CommandAck.AckType.Individual, new Dictionary <string, long>())); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult(); Assert.False(isDuplicate); tracker.Tell(new AddAcknowledgment(msg5, CommandAck.AckType.Cumulative, new Dictionary <string, long>())); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult(); Assert.False(isDuplicate); // Flush while disconnected. the internal tracking will not change tracker.Tell(FlushPending.Instance); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult(); Assert.False(isDuplicate); tracker.Tell(new AddAcknowledgment(msg6, CommandAck.AckType.Individual, new Dictionary <string, long>())); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult(); Assert.True(isDuplicate); tracker.Tell(FlushPending.Instance); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg1)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg2)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg3)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg4)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg5)).GetAwaiter().GetResult(); Assert.True(isDuplicate); isDuplicate = tracker.Ask <bool>(new IsDuplicate(msg6)).GetAwaiter().GetResult(); Assert.False(isDuplicate); tracker.GracefulStop(TimeSpan.FromSeconds(1)); }
public void TestGenericTopic() { var schema = AvroSchema <ComplexGenericData> .Of(typeof(ComplexGenericData)); var genericSchema = GenericAvroSchema.Of(schema.SchemaInfo); _output.WriteLine(schema.SchemaInfo.SchemaDefinition); var pBuilder = new ProducerConfigBuilder <IGenericRecord>() .Topic(_topic); var producer = _client.NewProducer(genericSchema, pBuilder); const int messageCount = 10; for (var i = 0; i < messageCount; i++) { var dataForWriter = new GenericRecord((Avro.RecordSchema)genericSchema.AvroSchema); dataForWriter.Add("Feature", "Education"); dataForWriter.Add("StringData", new Dictionary <string, string> { { "Index", i.ToString() }, { "FirstName", "Ebere" }, { "LastName", "Abanonu" } }); dataForWriter.Add("ComplexData", ToBytes(new ComplexData { ProductId = i, Point = i * 2, Sales = i * 2 * 5 })); var record = new GenericAvroRecord(null, genericSchema.AvroSchema, genericSchema.Fields, dataForWriter); var receipt = producer.Send(record); _output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions { WriteIndented = true })); } var messageReceived = 0; var builder = new ConsumerConfigBuilder <IGenericRecord>() .Topic(_topic) .ForceTopicCreation(true) .SubscriptionName($"generic_sub"); var consumer = _client.NewConsumer(ISchema <object> .AutoConsume(), builder); Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCount; ++i) { var m = consumer.Receive(); Assert.NotNull(m); var receivedMessage = m.Value; var feature = receivedMessage.GetField("Feature").ToString(); var strinData = (Dictionary <string, object>)receivedMessage.GetField("StringData"); var complexData = FromBytes <ComplexData>((byte[])receivedMessage.GetField("ComplexData")); _output.WriteLine(feature); _output.WriteLine(JsonSerializer.Serialize(strinData, new JsonSerializerOptions { WriteIndented = true })); _output.WriteLine(JsonSerializer.Serialize(complexData, new JsonSerializerOptions { WriteIndented = true })); messageReceived++; consumer.Acknowledge(m); } Assert.Equal(10, messageReceived); producer.Close(); consumer.Close(); }
public void TxnAckTestBatchedFailoverSub() { var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(normalTopic) .SubscriptionName($"test-{Guid.NewGuid()}") .ForceTopicCreation(true) .EnableBatchIndexAcknowledgment(true) .AcknowledgmentGroupTime(5000) .SubscriptionType(SubType.Failover); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(normalTopic) .EnableBatching(true) .BatchingMaxMessages(100); var producer = _client.NewProducer(producerBuilder); for (var retryCnt = 0; retryCnt < 1; retryCnt++) { var txn = Txn; //Thread.Sleep(TimeSpan.FromSeconds(30)); var messageCnt = 100; // produce normal messages for (var i = 0; i < messageCnt; i++) { producer.NewMessage().Value("hello".GetBytes()).Send(); } // consume and ack messages with txn Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { var msg = consumer.Receive(); Assert.NotNull(msg); _output.WriteLine($"receive msgId: {msg.MessageId}, count : {i}"); consumer.Acknowledge(msg.MessageId, txn); } // the messages are pending ack state and can't be received var message = consumer.Receive(); Assert.Null(message); // 1) txn abort txn.Abort(); // after transaction abort, the messages could be received var commitTxn = Txn; //Thread.Sleep(TimeSpan.FromSeconds(5)); Thread.Sleep(TimeSpan.FromSeconds(30)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message.MessageId, commitTxn); _output.WriteLine($"receive msgId: {message.MessageId}, count: {i}"); } // 2) ack committed by a new txn commitTxn.Commit(); // after transaction commit, the messages can't be received message = consumer.Receive(); Assert.Null(message); } }
public void TxnCumulativeAckTest() { var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(normalTopic) .SubscriptionName($"test-{Guid.NewGuid()}") .ForceTopicCreation(true) //.SubscriptionType(SubType.Failover) .EnableBatchIndexAcknowledgment(true) .AcknowledgmentGroupTime(3000) .AckTimeout(TimeSpan.FromMilliseconds(10000)); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(normalTopic); var producer = _client.NewProducer(producerBuilder); for (var retryCnt = 0; retryCnt < 2; retryCnt++) { var abortTxn = Txn; var messageCnt = 50; // produce normal messages for (var i = 0; i < messageCnt; i++) { producer.NewMessage().Value(Encoding.UTF8.GetBytes("Hello")).Send(); } IMessage <byte[]> message = null; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); if (i % 3 == 0) { consumer.AcknowledgeCumulative(message.MessageId, abortTxn); } _output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}"); } // the messages are pending ack state and can't be received message = consumer.Receive(); Assert.Null(message); abortTxn.Abort(); var commitTxn = Txn; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); consumer.AcknowledgeCumulative(message.MessageId, commitTxn); _output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}"); } commitTxn.Commit(); Thread.Sleep(TimeSpan.FromSeconds(5)); message = consumer.Receive(); Assert.Null(message); } }
private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout) { _output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}"); var topic = "testNegativeAcks-" + DateTime.Now.Ticks; var builder = new ConsumerConfigBuilder <byte[]>() .Topic(topic) .SubscriptionName($"sub1-{Guid.NewGuid()}") .AckTimeout(TimeSpan.FromMilliseconds(ackTimeout)) .ForceTopicCreation(true) .AcknowledgmentGroupTime(0) .NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis)) .SubscriptionType(subscriptionType); var consumer = _client.NewConsumer(builder); var pBuilder = new ProducerConfigBuilder <byte[]>(); pBuilder.Topic(topic); if (batching) { pBuilder.EnableBatching(batching); pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis)); pBuilder.BatchingMaxMessages(10); } var producer = _client.NewProducer(pBuilder); ISet <string> sentMessages = new HashSet <string>(); const int n = 10; for (var i = 0; i < n; i++) { var value = "test-" + i; producer.Send(Encoding.UTF8.GetBytes(value)); sentMessages.Add(value); } Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < n; i++) { var msg = consumer.Receive(); if (msg != null) { var ms = Encoding.UTF8.GetString(msg.Data); consumer.NegativeAcknowledge(msg); _output.WriteLine(ms); } } ISet <string> receivedMessages = new HashSet <string>(); Thread.Sleep(TimeSpan.FromSeconds(10)); // All the messages should be received again for (var i = 0; i < n; i++) { var msg = consumer.Receive(); if (msg != null) { var ms = Encoding.UTF8.GetString(msg.Data); _output.WriteLine(ms); receivedMessages.Add(ms); consumer.Acknowledge(msg); } } Assert.Equal(sentMessages, receivedMessages); var nu = consumer.Receive(); // There should be no more messages Assert.Null(nu); producer.Close(); consumer.Close(); }
public void TxnMessageAckTest() { var topic = $"{_topicMessageAckTest}-{Guid.NewGuid()}"; var subName = $"test-{Guid.NewGuid()}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(topic) .SubscriptionName(subName) .ForceTopicCreation(true) .EnableBatchIndexAcknowledgment(true) .AcknowledgmentGroupTime(0); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(topic) .EnableBatching(false) .SendTimeout(0); var producer = _client.NewProducer(producerBuilder); var txn = Txn; var messageCnt = 10; for (var i = 0; i < messageCnt; i++) { producer.NewMessage(txn).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send(); } _output.WriteLine("produce transaction messages finished"); // Can't receive transaction messages before commit. var message = consumer.Receive(); Assert.Null(message); _output.WriteLine("transaction messages can't be received before transaction committed"); txn.Commit(); var ackedMessageCount = 0; var receiveCnt = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); receiveCnt++; if (i % 2 == 0) { consumer.Acknowledge(message); ackedMessageCount++; } } Assert.Equal(messageCnt, receiveCnt); message = consumer.Receive(); Assert.Null(message); consumer.RedeliverUnacknowledgedMessages(); Thread.Sleep(TimeSpan.FromSeconds(10)); receiveCnt = 0; for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message); receiveCnt++; } Assert.Equal(messageCnt - ackedMessageCount, receiveCnt); message = consumer.Receive(); Assert.Null(message); _output.WriteLine($"receive transaction messages count: {receiveCnt}"); }
public virtual void TestBinaryProtoToGetTopicsOfNamespacePersistent() { var key = Guid.NewGuid().ToString(); var subscriptionName = "regex-subscription"; var topicName1 = "persistent://public/default/reg-topic-1-" + key; var topicName2 = "persistent://public/default/reg-topic-2-" + key; var topicName3 = "persistent://public/default/reg-topic-3-" + key; var topicName4 = "non-persistent://public/default/reg-topic-4-" + key; var pattern = new Regex("public/default/reg-topic*"); // 2. create producer var messagePredicate = "my-message-" + key + "-"; var producer1 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName1)); var producer2 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName2)); var producer3 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName3)); var producer4 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName4)); // 5. produce data for (var i = 0; i < 10; i++) { producer1.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer1-" + i)); producer2.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer2-" + i)); producer3.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer3-" + i)); producer4.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer4-" + i)); } // 6. should receive all the message var messageSet = 0; var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .TopicsPattern(pattern) .PatternAutoDiscoveryPeriod(2) .ForceTopicCreation(true) .SubscriptionName(subscriptionName) .SubscriptionType(SubType.Shared) .AckTimeout(TimeSpan.FromMilliseconds(60000))); Thread.Sleep(TimeSpan.FromSeconds(10)); var message = consumer.Receive(); if (message == null) { Thread.Sleep(TimeSpan.FromSeconds(10)); message = consumer.Receive(); } while (message != null) { var m = (TopicMessage <byte[]>)message; messageSet++; consumer.Acknowledge(m); _output.WriteLine($"Consumer acknowledged : {Encoding.UTF8.GetString(message.Data)} from topic: {m.Topic}"); message = consumer.Receive(); } consumer.Unsubscribe(); consumer.Close(); producer1.Close(); producer2.Close(); producer3.Close(); producer4.Close(); Assert.True(messageSet > 0); }
private void PlainAvroProducer(string topic) { var jsonSchem = AvroSchema <JournalEntry> .Of(typeof(JournalEntry)); var builder = new ConsumerConfigBuilder <JournalEntry>() .Topic(topic) .SubscriptionName($"my-subscriber-name-{DateTimeHelper.CurrentUnixTimeMillis()}") .AckTimeout(TimeSpan.FromMilliseconds(20000)) .ForceTopicCreation(true) .AcknowledgmentGroupTime(0); var consumer = _client.NewConsumer(jsonSchem, builder); var producerConfig = new ProducerConfigBuilder <JournalEntry>() .ProducerName(topic.Split("/").Last()) .Topic(topic) .Schema(jsonSchem) .SendTimeout(10000); var producer = _client.NewProducer(jsonSchem, producerConfig); for (var i = 0; i < 10; i++) { var student = new Students { Name = $"[{i}] Ebere: {DateTimeOffset.Now.ToUnixTimeMilliseconds()} - presto-ed {DateTime.Now.ToString(CultureInfo.InvariantCulture)}", Age = 202 + i, School = "Akka-Pulsar university" }; var journal = new JournalEntry { Id = $"[{i}]Ebere: {DateTimeOffset.Now.ToUnixTimeMilliseconds()}", PersistenceId = "sampleActor", IsDeleted = false, Ordering = 0, Payload = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(student)), SequenceNr = 0, Tags = "root" }; var metadata = new Dictionary <string, string> { ["Key"] = "Single", ["Properties"] = JsonSerializer.Serialize(new Dictionary <string, string> { { "Tick", DateTime.Now.Ticks.ToString() } }, new JsonSerializerOptions { WriteIndented = true }) }; var id = producer.NewMessage().Properties(metadata).Value(journal).Send(); } Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < 10; i++) { var msg = consumer.Receive(); if (msg != null) { var receivedMessage = msg.Value; _output.WriteLine(JsonSerializer.Serialize(receivedMessage, new JsonSerializerOptions { WriteIndented = true })); } } }
public void ProduceCommitTest() { var txn1 = Txn; var txn2 = Txn; var topic = $"{_topicOutput}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(topic) .ForceTopicCreation(true) .SubscriptionName($"test-{Guid.NewGuid()}"); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(topic) .SendTimeout(0); var producer = _client.NewProducer(producerBuilder); var txnMessageCnt = 0; var messageCnt = 10; for (var i = 0; i < messageCnt; i++) { if (i % 5 == 0) { producer.NewMessage(txn1).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send(); } else { producer.NewMessage(txn2).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send(); } txnMessageCnt++; } // Can't receive transaction messages before commit. var message = consumer.Receive(); Assert.Null(message); txn1.Commit(); _output.WriteLine($"Committed 1"); txn2.Commit(); _output.WriteLine($"Committed 1"); // txn1 messages could be received after txn1 committed var receiveCnt = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < txnMessageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); _output.WriteLine(Encoding.UTF8.GetString(message.Value)); receiveCnt++; } for (var i = 0; i < txnMessageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); receiveCnt++; } Assert.Equal(txnMessageCnt, receiveCnt); message = consumer.Receive(); Assert.Null(message); _output.WriteLine($"message commit test enableBatch {true}"); }