private ISet <string> PublishMessages(string topic, int count, bool enableBatch) { ISet <string> keys = new HashSet <string>(); var builder = new ProducerConfigBuilder <byte[]>() .Topic(topic) .MessageRoutingMode(Common.MessageRoutingMode.RoundRobinMode) .MaxPendingMessages(count) .BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(800000)); if (enableBatch) { builder.EnableBatching(true); builder.BatchingMaxMessages(count); } else { builder.EnableBatching(false); } var producer = _client.NewProducer(builder); for (var i = 0; i < count; i++) { var key = "key" + i; var data = Encoding.UTF8.GetBytes("my-message-" + i); producer.NewMessage().Key(key).Value(data).Send(); keys.Add(key); } producer.Flush(); return(keys); }
public void ProduceAndConsume() { var record1 = AvroSchema <SimpleRecord> .Of(typeof(SimpleRecord)); var consumerBuilder = new ConsumerConfigBuilder <SimpleRecord>() .Topic(_topic) .ConsumerName("avroUpgradeSchema1") .SubscriptionName("test-sub"); var consumer = _client.NewConsumer(record1, consumerBuilder); var producerBuilder = new ProducerConfigBuilder <SimpleRecord>() .Topic(_topic) .ProducerName("avroUpgradeSchema1"); var producer = _client.NewProducer(record1, producerBuilder); producer.NewMessage() .Value(new SimpleRecord { Name = "Ebere Abanonu", Age = int.MaxValue }) .Send(); Thread.Sleep(TimeSpan.FromSeconds(10)); var message = consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message); consumer.Unsubscribe(); var record2 = AvroSchema <SimpleRecord2> .Of(typeof(SimpleRecord2)); var consumerBuilder2 = new ConsumerConfigBuilder <SimpleRecord2>() .Topic(_topic) .ConsumerName("avroUpgradeSchema2") .SubscriptionName("test-sub"); var consumer1 = _client.NewConsumer(record2, consumerBuilder2); var producerBuilder2 = new ProducerConfigBuilder <SimpleRecord2>() .Topic(_topic) .ProducerName("avroUpgradeSchema2"); var producer2 = _client.NewProducer(record2, producerBuilder2); producer2.NewMessage() .Value(new SimpleRecord2 { Name = "Ebere", Age = int.MaxValue, Surname = "Abanonu" }) .Send(); Thread.Sleep(TimeSpan.FromSeconds(10)); var msg = consumer1.Receive(); Assert.NotNull(msg); consumer1.Acknowledge(msg); consumer1.Unsubscribe(); producer.Close(); producer2.Close(); consumer.Close(); consumer1.Close(); }
public void TestMultiTopic() { var topic = "persistent://public/default/topic" + Guid.NewGuid(); var topic2 = "persistent://public/default/topic2" + Guid.NewGuid(); var topic3 = "persistent://public/default/topic3" + Guid.NewGuid(); IList <string> topics = new List <string> { topic, topic2, topic3 }; var builder = new ReaderConfigBuilder <string>() .Topics(topics) .StartMessageId(IMessageId.Earliest) .ReaderName("my-reader"); var reader = _client.NewReader(ISchema <object> .String, builder); // create producer and send msg IList <Producer <string> > producerList = new List <Producer <string> >(); foreach (var topicName in topics) { var producer = _client.NewProducer(ISchema <object> .String, new ProducerConfigBuilder <string>().Topic(topicName)); producerList.Add(producer); } var msgNum = 10; ISet <string> messages = new HashSet <string>(); for (var i = 0; i < producerList.Count; i++) { var producer = producerList[i]; for (var j = 0; j < msgNum; j++) { var msg = i + "msg" + j; producer.Send(msg); messages.Add(msg); } } // receive messagesS var message = reader.ReadNext(TimeSpan.FromSeconds(5)); while (message != null) { var value = message.Value; _output.WriteLine(value); Assert.True(messages.Remove(value)); message = reader.ReadNext(TimeSpan.FromSeconds(5)); } Assert.Equal(0, messages.Count); // clean up foreach (var producer in producerList) { producer.Close(); } reader.Close(); }
public virtual void ProducerInstantiation() { var producer = new ProducerConfigBuilder <string>(); producer.Topic("ProducerInstantiation"); var stringProducerBuilder = _client.NewProducer(new StringSchema(), producer); Assert.NotNull(stringProducerBuilder); stringProducerBuilder.Close(); }
private ISet <MessageId> PublishMessages(string topic, int count) { var ids = new HashSet <MessageId>(); var builder = new ProducerConfigBuilder <DataOp>() .Topic(topic); var producer = _client.NewProducer(AvroSchema <DataOp> .Of(typeof(DataOp)), builder); for (var i = 0; i < count; i++) { var key = "key" + i; MessageId id = null; if (i % 2 == 0) { id = producer.NewMessage().Key(key).Property("twitter", "mestical").Value(new DataOp { Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis() }).Send(); } else { id = producer.NewMessage().Key(key).Value(new DataOp { Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis() }).Send(); } ids.Add(id); } return(ids); }
private static void ExclusiveProduceConsumer(PulsarClient pulsarClient) { var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>() .AccessMode(SharpPulsar.Common.ProducerAccessMode.Exclusive) .Topic(myTopic));; for (var i = 0; i < 100; i++) { var data = Encoding.UTF8.GetBytes($"tuts-{i}"); producer.NewMessage().Value(data).Send(); } var pool = ArrayPool <byte> .Shared; var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(myTopic) .ForceTopicCreation(true) .SubscriptionName("myTopic-sub-Exclusive") .SubscriptionInitialPosition(SharpPulsar.Common.SubscriptionInitialPosition.Earliest)); for (var i = 0; i < 100; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"message '{res}' from topic: {message.TopicName}"); } } }
private ISet <string> PublishMessages(string topic, int count) { ISet <string> keys = new HashSet <string>(); var builder = new ProducerConfigBuilder <DataOp>() .Topic(topic); var producer = _client.NewProducer(AvroSchema <DataOp> .Of(typeof(DataOp)), builder); for (var i = 0; i < count; i++) { var key = "key" + i; producer.NewMessage().Key(key).Value(new DataOp { Text = "my-sql-message-" + i }).Send(); keys.Add(key); } return(keys); }
public void ZeroQueueSizeNormalConsumer() { string key = "nonZeroQueueSizeNormalConsumer"; // 1. Config string topicName = "topic-" + key; string subscriptionName = "my-ex-subscription-" + key; string messagePredicate = "my-message-" + key + "-"; // 2. Create Producer var pBuilder = new ProducerConfigBuilder <sbyte[]>() .Topic(topicName) .EnableBatching(false); var producer = _client.NewProducer(pBuilder); // 3. Create Consumer var config = new ConsumerConfigBuilder <sbyte[]>() .Topic(topicName) .SubscriptionName(subscriptionName) .ReceiverQueueSize(0); var consumer = _client.NewConsumer(config); // 3. producer publish messages for (int i = 0; i < _totalMessages; i++) { var msg = messagePredicate + i; _output.WriteLine("Producer produced: " + msg); producer.Send(Encoding.UTF8.GetBytes(msg).ToSBytes()); } // 4. Receiver receives the message IMessage <sbyte[]> message; for (int i = 0; i < _totalMessages; i++) { Assert.Equal(0, consumer.NumMessagesInQueue()); message = consumer.Receive(); var r = Encoding.UTF8.GetString(message.Data.ToBytes()); Assert.Equal(r, messagePredicate + i); Assert.Equal(0, consumer.NumMessagesInQueue()); _output.WriteLine("Consumer received : " + r); } }
public void TestUnAckMessageRedeliveryWithReceive() { var topic = $"persistent://public/default/async-unack-redelivery-{Guid.NewGuid()}"; var builder = new ConsumerConfigBuilder <byte[]>(); builder.Topic(topic); builder.SubscriptionName("sub-TestUnAckMessageRedeliveryWithReceive"); builder.AckTimeout(TimeSpan.FromMilliseconds(8000)); builder.ForceTopicCreation(true); builder.AcknowledgmentGroupTime(0); builder.SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared); var consumer = _client.NewConsumer(builder); var pBuilder = new ProducerConfigBuilder <byte[]>(); pBuilder.Topic(topic); var producer = _client.NewProducer(pBuilder); const int messageCount = 10; for (var i = 0; i < messageCount; i++) { var receipt = producer.Send(Encoding.UTF8.GetBytes("my-message-" + i)); _output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions { WriteIndented = true })); } var messageReceived = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCount; ++i) { var m = consumer.Receive(); var receivedMessage = Encoding.UTF8.GetString(m.Data); _output.WriteLine($"Received message: [{receivedMessage}]"); Assert.NotNull(receivedMessage); messageReceived++; } Assert.Equal(10, messageReceived); Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < messageCount; i++) { var m = consumer.Receive(); var receivedMessage = Encoding.UTF8.GetString(m.Data); _output.WriteLine($"Received message: [{receivedMessage}]"); Assert.NotNull(receivedMessage); messageReceived++; } Assert.Equal(20, messageReceived); producer.Close(); consumer.Close(); }
private static void Transaction(PulsarClient pulsarClient) { var txn = (Transaction)pulsarClient.NewTransaction().WithTransactionTimeout(TimeSpan.FromMinutes(5)).Build(); var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>() .SendTimeout(0) .Topic(myTopic)); var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(myTopic) .ForceTopicCreation(true) .SubscriptionName("myTopic-sub")); for (var i = 0; i < 10; i++) { var text = $"tuts-{i}"; var data = Encoding.UTF8.GetBytes(text); producer.NewMessage(txn).Value(data).Send(); Console.WriteLine($"produced: {text}"); } var pool = ArrayPool <byte> .Shared; //Should not consume messages as the transaction is not committed yet for (var i = 0; i < 10; ++i) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"[1] message '{res}' from topic: {message.TopicName}"); } } txn.Commit(); for (var i = 0; i < 10; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"[2] message '{res}' from topic: {message.TopicName}"); } } }
public void TestDeliverAfter() { var numMessages = 5; var consumer = _client.NewConsumer(ISchema <string> .String, new ConsumerConfigBuilder <string>() .Topic(_topic) .SubscriptionName($"delayed-sub-{Guid.NewGuid()}") //deliverat works with shared subscription .SubscriptionType(Protocol.Proto.CommandSubscribe.SubType.Shared) .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); var producer = _client.NewProducer(ISchema <string> .String, new ProducerConfigBuilder <string>() .Topic(_topic)); // delay 5 seconds using DeliverAfter for (var i = 0; i < numMessages; i++) { producer.NewMessage().Value("DeliverAfter message " + i).DeliverAfter(TimeSpan.FromMilliseconds(5000)).Send(); } producer.Flush(); var numReceived = 0; while (numMessages <= 0 || numReceived < numMessages) { var msg = consumer.Receive(); if (msg == null) { Thread.Sleep(TimeSpan.FromSeconds(1)); continue; } var dt = (DateTimeHelper.CurrentUnixTimeMillis() - msg.PublishTime) / 1000; _output.WriteLine("Consumer Received message : " + msg.Value + "; Difference between publish time and receive time = " + dt + " seconds"); consumer.Acknowledge(msg); ++numReceived; } _output.WriteLine("Successfully received " + numReceived + " messages"); producer.Close(); consumer.Close(); }
public void A_100_000_producers() { for (var i = 0; i < 100_000; i++) { var p = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic($"A_100_000_producers-{i}-{Guid.NewGuid()}")); var data = Encoding.UTF8.GetBytes($"A_100_000_producers-{i}"); var id = p.NewMessage().Value(data).Send(); Console.WriteLine($"A_100_000_producers({id.LedgerId}:{id.EntryId})"); } Console.WriteLine("100,000 producers"); }
public void TestLargeMessage() { //this.conf.MaxMessageSize = 5; const int totalMessages = 3; var topicName = $"persistent://public/default/my-topic1-{DateTimeHelper.CurrentUnixTimeMillis()}"; var builder = new ConsumerConfigBuilder <byte[]>() .Topic(topicName) .SubscriptionName("my-subscriber-name") .AckTimeout(TimeSpan.FromMilliseconds(20000)) .ForceTopicCreation(true) .AcknowledgmentGroupTime(0); var consumer = _client.NewConsumer(builder); var pBuilder = new ProducerConfigBuilder <byte[]>() .Topic(topicName) .EnableChunking(true) .MaxMessageSize(5); var producer = _client.NewProducer(pBuilder); IList <string> publishedMessages = new List <string>(); for (var i = 1; i < totalMessages; i++) { var message = CreateMessagePayload(i * 10); publishedMessages.Add(message); producer.Send(Encoding.UTF8.GetBytes(message)); } IMessage <byte[]> msg = null; ISet <string> messageSet = new HashSet <string>(); IList <IMessage <byte[]> > msgIds = new List <IMessage <byte[]> >(); Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < totalMessages - 1; i++) { msg = consumer.Receive(); var receivedMessage = Encoding.UTF8.GetString(msg.Data); _output.WriteLine($"[{i}] - Published [{publishedMessages[i]}] Received message: [{receivedMessage}]"); var expectedMessage = publishedMessages[i]; TestMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage); msgIds.Add(msg); } foreach (var msgId in msgIds) { consumer.Acknowledge(msgId); } producer.Close(); consumer.Close(); }
public void Produce_Consumer_Allocation() { var topic = $"memory-allocation-{Guid.NewGuid()}"; var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(topic) .SubscriptionName($"myTopic-sub-{Guid.NewGuid()}") .IsAckReceiptEnabled(true) .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topic)); for (var i = 0; i < 500; i++) { var data = Encoding.UTF8.GetBytes($"memory-allocation-{i}"); producer.NewMessage().Value(data).Send(); } Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < 500; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); _output.WriteLine($"message '{res}' from topic: {message.Topic}"); } } dotMemory.Check(memory => { var result = memory .GetObjects(where => @where.Namespace.Like("SharpPulsar.*")); _output.WriteLine( ConsoleTable.From( result .GroupByType() .Select(t => new { t.Type.Name, t.ObjectsCount, t.SizeInBytes }) .OrderByDescending(x => x.SizeInBytes) ).ToMarkDownString()); Assert.True(result.ObjectsCount > 0); }); }
static async Task Publish() { var producer = await client.NewProducer(Schema.JSON <Test>()) .Topic(topicName) .CreateAsync(); while (running) { await producer.SendAsync(new Test { Text = DateTime.UtcNow.ToString() }); await Task.Delay(200); } //Console.WriteLine($"MessageId is: '{messageId}'"); }
private Producer <byte[]> CreateProducer(string topic, bool enableBatch, int batchSize = 500) { var pBuilder = new ProducerConfigBuilder <byte[]>(); pBuilder.Topic(topic); if (enableBatch) { pBuilder.EnableBatching(true); pBuilder.BatchBuilder(IBatcherBuilder.KeyBased(_client.ActorSystem)); pBuilder.BatchingMaxMessages(batchSize); pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(5000)); } return(_client.NewProducer(pBuilder)); }
private static List <MessageId> PublishMessages(string topic, int count, string message, PulsarClient client) { var keys = new List <MessageId>(); var builder = new ProducerConfigBuilder <byte[]>() .Topic(topic); var producer = client.NewProducer(builder); for (var i = 0; i < count; i++) { var key = "key" + i; var data = Encoding.UTF8.GetBytes($"{message}-{i}"); var id = producer.NewMessage().Key(key).Value(data).Send(); keys.Add(id); } return(keys); }
public void Setup() { var clientConfig = new PulsarClientConfigBuilder() .ServiceUrl("pulsar://localhost:6650"); _pulsarSystem = PulsarSystem.GetInstance(clientConfig); _client = _pulsarSystem.NewClient(); _consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(_benchTopic) .ForceTopicCreation(true) .SubscriptionName($"bench-sub-{Guid.NewGuid()}") .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); _producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(_benchTopic)); }
public void TestEncrptedProduceConsume() { var messageCount = 10; var topic = $"encrypted-messages-{Guid.NewGuid()}"; var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(topic) .ForceTopicCreation(true) .CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem")) .SubscriptionName("encrypted-sub") .SubscriptionInitialPosition(Common.SubscriptionInitialPosition.Earliest)); var producer = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topic) .CryptoKeyReader(new RawFileKeyReader("Certs/SharpPulsar_pub.pem", "Certs/SharpPulsar_private.pem")) .AddEncryptionKey("Ebere")); for (var i = 0; i < messageCount; i++) { producer.Send(Encoding.UTF8.GetBytes($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]")); } var receivedCount = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCount; i++) { var message = consumer.Receive(); if (message != null) { var decrypted = Encoding.UTF8.GetString(message.Data); _output.WriteLine(decrypted); Assert.Equal($"Shhhh, a secret: my is Ebere Abanonu and am a Nigerian based in Abeokuta, Ogun (a neighbouring State to Lagos - about 2 hours drive) [{i}]", decrypted); consumer.Acknowledge(message); receivedCount++; } } Assert.True(receivedCount > 6); producer.Close(); consumer.Close(); }
private static void ProduceConsumer(PulsarClient pulsarClient) { var consumer = pulsarClient.NewConsumer(new ConsumerConfigBuilder <byte[]>() .Topic(myTopic) .ForceTopicCreation(true) .SubscriptionName($"sub-{Guid.NewGuid()}") .IsAckReceiptEnabled(true)); var producer = pulsarClient.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(myTopic)); for (var i = 0; i < 1000; i++) { var data = Encoding.UTF8.GetBytes($"tuts-{i}"); var id = producer.NewMessage().Value(data).Send(); Console.WriteLine($"Message Id({id.LedgerId}:{id.EntryId})"); } var pool = ArrayPool <byte> .Shared; Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < 1000; i++) { var message = (Message <byte[]>)consumer.Receive(); if (message != null) { var payload = pool.Rent((int)message.Data.Length); Array.Copy(sourceArray: message.Data.ToArray(), destinationArray: payload, length: (int)message.Data.Length); consumer.Acknowledge(message); var res = Encoding.UTF8.GetString(message.Data); Console.WriteLine($"message '{res}' from topic: {message.TopicName}"); } } }
public void TestGenericTopic() { var schema = AvroSchema <ComplexGenericData> .Of(typeof(ComplexGenericData)); var genericSchema = GenericAvroSchema.Of(schema.SchemaInfo); _output.WriteLine(schema.SchemaInfo.SchemaDefinition); var pBuilder = new ProducerConfigBuilder <IGenericRecord>() .Topic(_topic); var producer = _client.NewProducer(genericSchema, pBuilder); const int messageCount = 10; for (var i = 0; i < messageCount; i++) { var dataForWriter = new GenericRecord((Avro.RecordSchema)genericSchema.AvroSchema); dataForWriter.Add("Feature", "Education"); dataForWriter.Add("StringData", new Dictionary <string, string> { { "Index", i.ToString() }, { "FirstName", "Ebere" }, { "LastName", "Abanonu" } }); dataForWriter.Add("ComplexData", ToBytes(new ComplexData { ProductId = i, Point = i * 2, Sales = i * 2 * 5 })); var record = new GenericAvroRecord(null, genericSchema.AvroSchema, genericSchema.Fields, dataForWriter); var receipt = producer.Send(record); _output.WriteLine(JsonSerializer.Serialize(receipt, new JsonSerializerOptions { WriteIndented = true })); } var messageReceived = 0; var builder = new ConsumerConfigBuilder <IGenericRecord>() .Topic(_topic) .ForceTopicCreation(true) .SubscriptionName($"generic_sub"); var consumer = _client.NewConsumer(ISchema <object> .AutoConsume(), builder); Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCount; ++i) { var m = consumer.Receive(); Assert.NotNull(m); var receivedMessage = m.Value; var feature = receivedMessage.GetField("Feature").ToString(); var strinData = (Dictionary <string, object>)receivedMessage.GetField("StringData"); var complexData = FromBytes <ComplexData>((byte[])receivedMessage.GetField("ComplexData")); _output.WriteLine(feature); _output.WriteLine(JsonSerializer.Serialize(strinData, new JsonSerializerOptions { WriteIndented = true })); _output.WriteLine(JsonSerializer.Serialize(complexData, new JsonSerializerOptions { WriteIndented = true })); messageReceived++; consumer.Acknowledge(m); } Assert.Equal(10, messageReceived); producer.Close(); consumer.Close(); }
public void TxnAckTestBatchedFailoverSub() { var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(normalTopic) .SubscriptionName($"test-{Guid.NewGuid()}") .ForceTopicCreation(true) .EnableBatchIndexAcknowledgment(true) .AcknowledgmentGroupTime(5000) .SubscriptionType(SubType.Failover); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(normalTopic) .EnableBatching(true) .BatchingMaxMessages(100); var producer = _client.NewProducer(producerBuilder); for (var retryCnt = 0; retryCnt < 1; retryCnt++) { var txn = Txn; //Thread.Sleep(TimeSpan.FromSeconds(30)); var messageCnt = 100; // produce normal messages for (var i = 0; i < messageCnt; i++) { producer.NewMessage().Value("hello".GetBytes()).Send(); } // consume and ack messages with txn Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { var msg = consumer.Receive(); Assert.NotNull(msg); _output.WriteLine($"receive msgId: {msg.MessageId}, count : {i}"); consumer.Acknowledge(msg.MessageId, txn); } // the messages are pending ack state and can't be received var message = consumer.Receive(); Assert.Null(message); // 1) txn abort txn.Abort(); // after transaction abort, the messages could be received var commitTxn = Txn; //Thread.Sleep(TimeSpan.FromSeconds(5)); Thread.Sleep(TimeSpan.FromSeconds(30)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message.MessageId, commitTxn); _output.WriteLine($"receive msgId: {message.MessageId}, count: {i}"); } // 2) ack committed by a new txn commitTxn.Commit(); // after transaction commit, the messages can't be received message = consumer.Receive(); Assert.Null(message); } }
public void TxnCumulativeAckTest() { var normalTopic = _nAMESPACE1 + $"/normal-topic-{Guid.NewGuid()}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(normalTopic) .SubscriptionName($"test-{Guid.NewGuid()}") .ForceTopicCreation(true) //.SubscriptionType(SubType.Failover) .EnableBatchIndexAcknowledgment(true) .AcknowledgmentGroupTime(3000) .AckTimeout(TimeSpan.FromMilliseconds(10000)); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(normalTopic); var producer = _client.NewProducer(producerBuilder); for (var retryCnt = 0; retryCnt < 2; retryCnt++) { var abortTxn = Txn; var messageCnt = 50; // produce normal messages for (var i = 0; i < messageCnt; i++) { producer.NewMessage().Value(Encoding.UTF8.GetBytes("Hello")).Send(); } IMessage <byte[]> message = null; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); if (i % 3 == 0) { consumer.AcknowledgeCumulative(message.MessageId, abortTxn); } _output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}"); } // the messages are pending ack state and can't be received message = consumer.Receive(); Assert.Null(message); abortTxn.Abort(); var commitTxn = Txn; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); consumer.AcknowledgeCumulative(message.MessageId, commitTxn); _output.WriteLine($"receive msgId abort: {message.MessageId}, retryCount : {retryCnt}, count : {i}"); } commitTxn.Commit(); Thread.Sleep(TimeSpan.FromSeconds(5)); message = consumer.Receive(); Assert.Null(message); } }
private void TestNegativeAcks(bool batching, bool usePartition, CommandSubscribe.SubType subscriptionType, int negAcksDelayMillis, int ackTimeout) { _output.WriteLine($"Test negative acks batching={batching} partitions={usePartition} subType={subscriptionType} negAckDelayMs={negAcksDelayMillis}"); var topic = "testNegativeAcks-" + DateTime.Now.Ticks; var builder = new ConsumerConfigBuilder <byte[]>() .Topic(topic) .SubscriptionName($"sub1-{Guid.NewGuid()}") .AckTimeout(TimeSpan.FromMilliseconds(ackTimeout)) .ForceTopicCreation(true) .AcknowledgmentGroupTime(0) .NegativeAckRedeliveryDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis)) .SubscriptionType(subscriptionType); var consumer = _client.NewConsumer(builder); var pBuilder = new ProducerConfigBuilder <byte[]>(); pBuilder.Topic(topic); if (batching) { pBuilder.EnableBatching(batching); pBuilder.BatchingMaxPublishDelay(TimeSpan.FromMilliseconds(negAcksDelayMillis)); pBuilder.BatchingMaxMessages(10); } var producer = _client.NewProducer(pBuilder); ISet <string> sentMessages = new HashSet <string>(); const int n = 10; for (var i = 0; i < n; i++) { var value = "test-" + i; producer.Send(Encoding.UTF8.GetBytes(value)); sentMessages.Add(value); } Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < n; i++) { var msg = consumer.Receive(); if (msg != null) { var ms = Encoding.UTF8.GetString(msg.Data); consumer.NegativeAcknowledge(msg); _output.WriteLine(ms); } } ISet <string> receivedMessages = new HashSet <string>(); Thread.Sleep(TimeSpan.FromSeconds(10)); // All the messages should be received again for (var i = 0; i < n; i++) { var msg = consumer.Receive(); if (msg != null) { var ms = Encoding.UTF8.GetString(msg.Data); _output.WriteLine(ms); receivedMessages.Add(ms); consumer.Acknowledge(msg); } } Assert.Equal(sentMessages, receivedMessages); var nu = consumer.Receive(); // There should be no more messages Assert.Null(nu); producer.Close(); consumer.Close(); }
public void TxnMessageAckTest() { var topic = $"{_topicMessageAckTest}-{Guid.NewGuid()}"; var subName = $"test-{Guid.NewGuid()}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(topic) .SubscriptionName(subName) .ForceTopicCreation(true) .EnableBatchIndexAcknowledgment(true) .AcknowledgmentGroupTime(0); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(topic) .EnableBatching(false) .SendTimeout(0); var producer = _client.NewProducer(producerBuilder); var txn = Txn; var messageCnt = 10; for (var i = 0; i < messageCnt; i++) { producer.NewMessage(txn).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send(); } _output.WriteLine("produce transaction messages finished"); // Can't receive transaction messages before commit. var message = consumer.Receive(); Assert.Null(message); _output.WriteLine("transaction messages can't be received before transaction committed"); txn.Commit(); var ackedMessageCount = 0; var receiveCnt = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); receiveCnt++; if (i % 2 == 0) { consumer.Acknowledge(message); ackedMessageCount++; } } Assert.Equal(messageCnt, receiveCnt); message = consumer.Receive(); Assert.Null(message); consumer.RedeliverUnacknowledgedMessages(); Thread.Sleep(TimeSpan.FromSeconds(10)); receiveCnt = 0; for (var i = 0; i < messageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message); receiveCnt++; } Assert.Equal(messageCnt - ackedMessageCount, receiveCnt); message = consumer.Receive(); Assert.Null(message); _output.WriteLine($"receive transaction messages count: {receiveCnt}"); }
public void ProduceCommitTest() { var txn1 = Txn; var txn2 = Txn; var topic = $"{_topicOutput}"; var consumerBuilder = new ConsumerConfigBuilder <byte[]>() .Topic(topic) .ForceTopicCreation(true) .SubscriptionName($"test-{Guid.NewGuid()}"); var consumer = _client.NewConsumer(consumerBuilder); var producerBuilder = new ProducerConfigBuilder <byte[]>() .Topic(topic) .SendTimeout(0); var producer = _client.NewProducer(producerBuilder); var txnMessageCnt = 0; var messageCnt = 10; for (var i = 0; i < messageCnt; i++) { if (i % 5 == 0) { producer.NewMessage(txn1).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send(); } else { producer.NewMessage(txn2).Value(Encoding.UTF8.GetBytes("Hello Txn - " + i)).Send(); } txnMessageCnt++; } // Can't receive transaction messages before commit. var message = consumer.Receive(); Assert.Null(message); txn1.Commit(); _output.WriteLine($"Committed 1"); txn2.Commit(); _output.WriteLine($"Committed 1"); // txn1 messages could be received after txn1 committed var receiveCnt = 0; Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < txnMessageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); _output.WriteLine(Encoding.UTF8.GetString(message.Value)); receiveCnt++; } for (var i = 0; i < txnMessageCnt; i++) { message = consumer.Receive(); Assert.NotNull(message); receiveCnt++; } Assert.Equal(txnMessageCnt, receiveCnt); message = consumer.Receive(); Assert.Null(message); _output.WriteLine($"message commit test enableBatch {true}"); }
public virtual void TestBinaryProtoToGetTopicsOfNamespacePersistent() { var key = Guid.NewGuid().ToString(); var subscriptionName = "regex-subscription"; var topicName1 = "persistent://public/default/reg-topic-1-" + key; var topicName2 = "persistent://public/default/reg-topic-2-" + key; var topicName3 = "persistent://public/default/reg-topic-3-" + key; var topicName4 = "non-persistent://public/default/reg-topic-4-" + key; var pattern = new Regex("public/default/reg-topic*"); // 2. create producer var messagePredicate = "my-message-" + key + "-"; var producer1 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName1)); var producer2 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName2)); var producer3 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName3)); var producer4 = _client.NewProducer(new ProducerConfigBuilder <byte[]>() .Topic(topicName4)); // 5. produce data for (var i = 0; i < 10; i++) { producer1.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer1-" + i)); producer2.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer2-" + i)); producer3.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer3-" + i)); producer4.Send(Encoding.UTF8.GetBytes(messagePredicate + "producer4-" + i)); } // 6. should receive all the message var messageSet = 0; var consumer = _client.NewConsumer(new ConsumerConfigBuilder <byte[]>() .TopicsPattern(pattern) .PatternAutoDiscoveryPeriod(2) .ForceTopicCreation(true) .SubscriptionName(subscriptionName) .SubscriptionType(SubType.Shared) .AckTimeout(TimeSpan.FromMilliseconds(60000))); Thread.Sleep(TimeSpan.FromSeconds(10)); var message = consumer.Receive(); if (message == null) { Thread.Sleep(TimeSpan.FromSeconds(10)); message = consumer.Receive(); } while (message != null) { var m = (TopicMessage <byte[]>)message; messageSet++; consumer.Acknowledge(m); _output.WriteLine($"Consumer acknowledged : {Encoding.UTF8.GetString(message.Data)} from topic: {m.Topic}"); message = consumer.Receive(); } consumer.Unsubscribe(); consumer.Close(); producer1.Close(); producer2.Close(); producer3.Close(); producer4.Close(); Assert.True(messageSet > 0); }