Example #1
0
        public bool Produce(ProduceMessage message)
        {
            MessageReceived(message.Topic);
            var ack = new ProduceAcknowledgement
            {
                OriginalBatch   = new TestBatchByTopicByPartition(new[] { message }),
                ProduceResponse =
                    new CommonResponse <ProducePartitionResponse>()
                {
                    TopicsResponse =
                        new[]
                    {
                        new TopicData <ProducePartitionResponse>
                        {
                            TopicName      = message.Topic,
                            PartitionsData =
                                new[]
                            {
                                new ProducePartitionResponse
                                {
                                    ErrorCode = ErrorCode.NoError,
                                    Offset    = 0,
                                    Partition = message.Partition
                                }
                            }
                        }
                    }
                },
                ReceiveDate = DateTime.UtcNow
            };

            ProduceAcknowledgement(this, ack);
            return(true);
        }
Example #2
0
        public void TestRoundRobinPartitionWithStartSeed(int startSeed, int delay)
        {
            var nodeMock   = new NodeMock();
            var partitions = new[]
            {
                new Partition {
                    Id = 0, Leader = nodeMock
                },
                new Partition {
                    Id = 1, Leader = nodeMock
                },
                new Partition {
                    Id = 2, Leader = nodeMock
                },
                new Partition {
                    Id = 3, Leader = nodeMock
                },
                new Partition {
                    Id = 4, Leader = nodeMock
                },
            };
            var partitionStrategy = new RoundRobinPartitionSelection(delay: delay, startSeed: startSeed);
            var partitioner       = new PartitionSelector(partitionStrategy);

            foreach (var partition in partitions)
            {
                for (var j = 0; j < delay; ++j)
                {
                    Assert.AreEqual((partition.Id + startSeed) % partitions.Length, partitioner.GetPartition(
                                        ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions).Id);
                }
            }
        }
Example #3
0
        public void TestRoundRobinPartitionAssign(int delay)
        {
            var nodeMock   = new NodeMock();
            var partitions = new[]
            {
                new Partition {
                    Id = 0, Leader = nodeMock
                },
                new Partition {
                    Id = 1, Leader = nodeMock
                },
                new Partition {
                    Id = 2, Leader = nodeMock
                },
                new Partition {
                    Id = 3, Leader = nodeMock
                },
                new Partition {
                    Id = 4, Leader = nodeMock
                },
            };
            var partitionStrategy = new RoundRobinPartitionSelection(delay);
            var partitioner       = new PartitionSelector(partitionStrategy);

            delay = delay <= 0 ? 1 : delay;
            foreach (var partition in partitions)
            {
                for (var j = 0; j < delay; ++j)
                {
                    Assert.AreEqual(partition.Id, partitioner
                                    .GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions)
                                    .Id);
                }
            }
        }
Example #4
0
        public void TestNodeFailToConnectReRouteMessages()
        {
            int rec    = 0;
            var ev     = new ManualResetEvent(false);
            var config = new Configuration {
                ProduceBatchSize = 1, ProduceBufferingTime = TimeSpan.FromMilliseconds(15)
            };
            var node =
                new Node("[Failing node]", () => new ConnectFailingConnectionMock(), new DummySerialization(), config, 1);
            bool dead = false;

            node.Dead += _ =>
            {
                Assert.AreEqual(node, _);
                dead = true;
            };
            node.ProduceAcknowledgement += (n, ack) =>
            {
                Assert.AreSame(node, n);
                if (Interlocked.Add(ref rec, ack.OriginalBatch.SelectMany(g => g).Count()) == 2)
                {
                    ev.Set();
                }
            };

            node.Produce(ProduceMessage.New("poulpe", 1, new Message(), DateTime.UtcNow.AddMinutes(5)));
            node.Produce(ProduceMessage.New("poulpe", 2, new Message(), DateTime.UtcNow.AddMinutes(5)));

            ev.WaitOne();
            Assert.IsTrue(dead);
        }
Example #5
0
        public async Task TestAcknowledgementResponseNoneProduceWasSentDiscard()
        {
            _finished = new AsyncCountdownEvent(1);
            int discarded = 0;

            _produceRouter.MessageDiscarded += (t, m) =>
            {
                discarded += 1;
                _finished.Signal();
            };
            var acknowledgement = new ProduceAcknowledgement
            {
                OriginalBatch = new TestBatchByTopicByPartition(new[]
                {
                    ProduceMessage.New("test1p", 0, new Message(),
                                       DateTime.UtcNow.AddDays(1))
                }),
                ReceiveDate = DateTime.UtcNow
            };

            _produceRouter.Acknowledge(acknowledgement);
            await _finished.WaitAsync();

            Assert.AreEqual(1, discarded);
        }
Example #6
0
        public void TestProduceWithNoErrorsNoAck()
        {
            var node = new Node("Node", () => new EchoConnectionMock(), new ProduceSerialization(new CommonResponse <ProducePartitionResponse>()),
                                new Configuration {
                ProduceBufferingTime = TimeSpan.FromMilliseconds(15), RequiredAcks = RequiredAcks.None
            }, 1);
            var  count = new CountdownEvent(2);
            bool batch = false;
            ProduceAcknowledgement acknowledgement = new ProduceAcknowledgement();

            node.ProduceBatchSent += (n, c, s) =>
            {
                count.Signal();
            };
            node.ProduceAcknowledgement += (n, ack) =>
            {
                acknowledgement = ack;
                count.Signal();
            };

            node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));

            count.Wait();
            Assert.AreEqual(ErrorCode.NoError, acknowledgement.ProduceResponse.TopicsResponse[0].PartitionsData.First().ErrorCode);
        }
Example #7
0
        public void TestProduceReplicaNotAvailableIsNotAnError()
        {
            var acknowledgement = new ProduceAcknowledgement
            {
                ProduceResponse = new CommonResponse <ProducePartitionResponse>
                {
                    TopicsResponse = new[]
                    {
                        new TopicData <ProducePartitionResponse>
                        {
                            TopicName      = "test",
                            PartitionsData = new[]
                            {
                                new ProducePartitionResponse
                                {
                                    ErrorCode = ErrorCode.ReplicaNotAvailable,
                                    Offset    = 0,
                                    Partition = 0
                                }
                            }
                        }
                    }
                },
                OriginalBatch =
                    new TestBatchByTopicByPartition(new[]
                {
                    ProduceMessage.New("test", 0, new Message(),
                                       DateTime.UtcNow.AddDays(1))
                })
            };

            _TestAcknowledgementNoError(acknowledgement, 1);
        }
Example #8
0
        public void TestProduceWithNoErrors()
        {
            double requestLatency  = 0;
            var    expectedLatency = 3;
            var    node            = new Node("Node", () => new EchoConnectionMock(false, expectedLatency), new ProduceSerialization(new CommonResponse <ProducePartitionResponse>()),
                                              new Configuration {
                ProduceBufferingTime = TimeSpan.FromMilliseconds(15)
            }, 1);
            var count = new CountdownEvent(2);

            node.ResponseReceived += (n, l) =>
            {
                Assert.AreSame(node, n);
                requestLatency = l;
                count.Signal();
            };
            bool batch = false;

            node.ProduceBatchSent += (n, c, s) =>
            {
                Assert.That(n, Is.EqualTo(node));
                Assert.That(c, Is.EqualTo(1));
                Assert.That(s, Is.EqualTo(0));
                batch = true;
                count.Signal();
            };

            node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));

            count.Wait();
            Assert.IsTrue(batch);
            Assert.GreaterOrEqual(requestLatency, expectedLatency);
        }
Example #9
0
        public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_Partition_Blacklisted(int partitionIdBlacklisted)
        {
            var nodeMock   = new NodeMock();
            var partitions = new[]
            {
                new Partition {
                    Id = 0, Leader = nodeMock
                },
                new Partition {
                    Id = 1, Leader = nodeMock
                },
                new Partition {
                    Id = 2, Leader = nodeMock
                },
            };
            var blacklistedPartitions = new Dictionary <int, DateTime> {
                { partitionIdBlacklisted, DateTime.MaxValue }
            };
            var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
            var partitioner       = new PartitionSelector(partitionStrategy);
            var message           = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
                Key = "ThisIsMyKey"
            }, new DateTime());

            for (var i = 0; i < 300; i++)
            {
                var partition = partitioner.GetPartition(message, partitions, blacklistedPartitions);
                Assert.IsTrue(partition.Id != Partition.None.Id);
                Assert.IsTrue(partition.Id != partitionIdBlacklisted);
            }
        }
Example #10
0
        public void Test_MessageKeyPartitionSelection_Fallbacks_To_RoundRobin_If_MessageKey_Null()
        {
            var nodeMock   = new NodeMock();
            var partitions = new[]
            {
                new Partition {
                    Id = 0, Leader = nodeMock
                },
                new Partition {
                    Id = 1, Leader = nodeMock
                },
                new Partition {
                    Id = 2, Leader = nodeMock
                },
            };
            var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
            var partitioner       = new PartitionSelector(partitionStrategy);
            var message           = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
                Key = null
            }, new DateTime());

            var partition = partitioner.GetPartition(message, partitions);

            Assert.IsTrue(partition.Id != Partition.None.Id);
        }
Example #11
0
        public void Test_MessageKeyPartitionSelection_Is_Consistent()
        {
            var nodeMock   = new NodeMock();
            var partitions = new[]
            {
                new Partition {
                    Id = 0, Leader = nodeMock
                },
                new Partition {
                    Id = 1, Leader = nodeMock
                },
                new Partition {
                    Id = 2, Leader = nodeMock
                },
            };
            var partitionStrategy = new MessageKeyPartitionSelection(Serializer, RoundRobinPartitionSelection, Mock.Of <ILogger>());
            var partitioner       = new PartitionSelector(partitionStrategy);
            var message1          = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
                Key = "ThisIsMyKey"
            }, new DateTime());
            var message2 = ProduceMessage.New(string.Empty, Partitions.Any, new Message {
                Key = "ThisIsMyOtherKey"
            }, new DateTime());

            var expectedPartition1 = partitioner.GetPartition(message1, partitions);
            var expectedPartition2 = partitioner.GetPartition(message2, partitions);

            for (var i = 0; i < 300; i++)
            {
                var currentPartition1 = partitioner.GetPartition(message1, partitions);
                var currentPartition2 = partitioner.GetPartition(message2, partitions);
                Assert.AreEqual(expectedPartition1.Id, currentPartition1.Id);
                Assert.AreEqual(expectedPartition2.Id, currentPartition2.Id);
            }
        }
Example #12
0
        public void TestRobinPartitionAssignWhenFiltered()
        {
            var nodeMock = new NodeMock();

            var partitions = new[]
            {
                new Partition {
                    Id = 1, Leader = nodeMock
                },
                new Partition {
                    Id = 2, Leader = nodeMock
                },
                new Partition {
                    Id = 3, Leader = nodeMock
                },
            };

            var filter = new Dictionary <int, DateTime>();

            int delay = partitions.Length + 2;

            var partitionStrategy = new RoundRobinPartitionSelection(delay);
            var partitioner       = new PartitionSelector(partitionStrategy);

            var partition = partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter);

            Assert.AreEqual(1, partition.Id);

            filter.Add(1, DateTime.UtcNow);

            var batch = GetPartitions(delay, partitioner, partitions, filter);

            Assert.AreEqual(delay, batch.Count);
            Assert.IsTrue(batch.All(p => p.Id == 2), "The round-robin threshold wasn't properly reset after previous partition was blacklisted");
        }
        private RabbitMQRequest(long chatId, string message)
        {
            IProduceMessage produceMessage = new ProduceMessage();

            produceMessage.ProduceMessageString(Environment.GetEnvironmentVariable("queueName"), JsonConvert.SerializeObject(new Payload()
            {
                ChatId = chatId, Message = message
            }));
        }
Example #14
0
        // Raise the MessageExpired event and release a message.
        private void OnMessageExpired(ProduceMessage message)
        {
            _cluster.Logger.LogError(string.Format(
                                         "[Producer] Not able to send message before reaching TTL for [topic: {0} / partition: {1}], message expired.",
                                         message.Topic, message.RequiredPartition));

            message.Message = CheckReleaseMessage(message.Message);
            MessageExpired(message.Topic, message.Message);
        }
Example #15
0
        // Raise the MessageExpired event and release a message.
        private void OnMessageExpired(ProduceMessage message)
        {
            _cluster.Logger.LogError(string.Format(
                                         "[Producer] Not able to send message before reaching TTL for [topic: {0} / partition: {1}], message expired.",
                                         message.Topic, message.RequiredPartition));

            ClearMessage(message.Message, shouldClearKeyValue: false);
            MessageExpired(message.Topic, message.Message);
        }
Example #16
0
        public void TestRoundRobinPartitionAssignNoPartitionReturnsNone()
        {
            var partitions        = new Partition[0];
            var partitionStrategy = new RoundRobinPartitionSelection();
            var partitioner       = new PartitionSelector(partitionStrategy);

            Assert.AreEqual(0, Partition.None.CompareTo(partitioner.GetPartition(
                                                            ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions)));
        }
Example #17
0
 /// <summary>
 /// Prepare a new message for routing.
 /// </summary>
 /// <param name="topic"></param>
 /// <param name="message"></param>
 /// <param name="partition"></param>
 /// <param name="expirationDate"></param>
 public void Route(string topic, Message message, int partition, DateTime expirationDate)
 {
     if (_configuration.SerializationConfig.SerializeOnProduce)
     {
         var serializers = _configuration.SerializationConfig.GetSerializersForTopic(topic);
         message.SerializeKeyValue(_pool.Reserve(), serializers);
     }
     Route(ProduceMessage.New(topic, partition, message, expirationDate));
 }
Example #18
0
        public void TestAcknowledgementMultipleNoError()
        {
            var acknowledgement = new ProduceAcknowledgement
            {
                ProduceResponse = new ProduceResponse {
                    ProducePartitionResponse = new CommonResponse <ProducePartitionResponse>()
                    {
                        TopicsResponse = new[]
                        {
                            new TopicData <ProducePartitionResponse>
                            {
                                TopicName      = "test",
                                PartitionsData = new[]
                                {
                                    new ProducePartitionResponse
                                    {
                                        ErrorCode = ErrorCode.NoError,
                                        Offset    = 0,
                                        Partition = 0
                                    },
                                    new ProducePartitionResponse
                                    {
                                        ErrorCode = ErrorCode.NoError,
                                        Offset    = 0,
                                        Partition = 1
                                    },
                                    new ProducePartitionResponse
                                    {
                                        ErrorCode = ErrorCode.NoError,
                                        Offset    = 0,
                                        Partition = 2
                                    }
                                }
                            }
                        }
                    }
                },
                OriginalBatch =
                    new TestBatchByTopicByPartition(new[]
                {
                    ProduceMessage.New("test", 0, new Message(),
                                       DateTime.UtcNow.AddDays(1)),
                    ProduceMessage.New("test", 1, new Message(),
                                       DateTime.UtcNow.AddDays(1)),
                    ProduceMessage.New("test", 2, new Message(),
                                       DateTime.UtcNow.AddDays(1)),
                    ProduceMessage.New("test2", 0, new Message(),
                                       DateTime.UtcNow.AddDays(1)),
                    ProduceMessage.New("test2", 1, new Message(),
                                       DateTime.UtcNow.AddDays(1))
                })
            };

            _TestAcknowledgementNoError(acknowledgement, 5);
        }
Example #19
0
        private static List <Partition> GetPartitions(int count, PartitionSelector partitioner, Partition[] partitions, Dictionary <int, DateTime> filter)
        {
            var result = new List <Partition>(count);

            for (int i = 0; i < count; i++)
            {
                result.Add(partitioner.GetPartition(ProduceMessage.New(string.Empty, Partitions.Any, new Message(), new DateTime()), partitions, filter));
            }

            return(result);
        }
Example #20
0
        public void TestMaxRetry()
        {
            int discarded = 0;

            _produceRouter.MessageDiscarded += (t, m) => discarded += 1;
            var pm = ProduceMessage.New("test1p", Partitions.Any, new Message(), DateTime.UtcNow.AddMinutes(5));

            pm.Retried = 42;
            _produceRouter.ReEnqueue(pm);
            Assert.AreEqual(1, discarded);
        }
        public Partition GetPartition(ProduceMessage produceMessage, Partition[] partitions, IReadOnlyDictionary <int, DateTime> blacklist)
        {
            var partitionId = GetPartitionIdFromKey(produceMessage.Message.Key, partitions.Length);

            if (partitionId != Partition.None.Id && !blacklist.ContainsKey(partitionId))
            {
                return(partitions[partitionId]);
            }

            return(_roundRobinSelection.GetPartition(produceMessage, partitions, blacklist));
        }
Example #22
0
 /// <summary>
 /// Inner Post produce implementation. Post a ProduceEvent message
 /// to the inner actor.
 /// </summary>
 /// <param name="produceMessage"></param>
 /// <returns>True if effectiveley posted, false otherwise</returns>
 private bool Post(ProduceMessage produceMessage)
 {
     if (_messages.Completion.IsCompleted)
     {
         return(false);
     }
     _produceMessages.Enqueue(produceMessage);
     // There is a race here, since Stop can happen in between the
     // previous and next lines. However this only happen when Stop
     // is required which means we can safely let a few messages in
     // the queue, they will be signaled as discarded to the outer
     // world (Stop is permanent).
     return(_messages.Post(RouterMessageType.ProduceEvent));
 }
Example #23
0
 /// <summary>
 /// Try to send again a message following an error.
 /// Check for retry if needed.
 /// </summary>
 internal void ReEnqueueAfterError(ProduceMessage message)
 {
     if (_configuration.MaxRetry < 0 || ++message.Retried <= _configuration.MaxRetry)
     {
         ReEnqueueMessage(message);
     }
     else
     {
         _cluster.Logger.LogError(string.Format(
                                      "[Producer] Not able to reenqueue: too many retry ({1}). Discarding message for topic '{0}'",
                                      message.Topic, message.Retried));
         OnMessageDiscarded(message);
     }
 }
Example #24
0
        public async Task TestAcknowledgementResponseNoneProduceWasNotSent()
        {
            _finished = new AsyncCountdownEvent(3);
            var acknowledgement = new ProduceAcknowledgement
            {
                OriginalBatch =
                    new TestBatchByTopicByPartition(new[]
                                                    { ProduceMessage.New("test1p", 0, new Message(), DateTime.UtcNow.AddDays(1)) })
            };

            _produceRouter.Acknowledge(acknowledgement);
            await _finished.WaitAsync();

            CheckCounters(expectedMessagesReEnqueued: 1, expectedMessagesRouted: 1, expectedRoutingTableRequired: 1);
        }
Example #25
0
        public async Task <object> ProduceMessage([FromBody] ProduceMessage produceMessage)
        {
            var producer = GetProducer(produceMessage.Broker);
            var result   = await producer.ProduceAsync(produceMessage.Topic, new Message <string, string>
            {
                Key   = produceMessage.Key,
                Value = new
                {
                    Headers = produceMessage.MessageHeaders,
                    Payload = produceMessage.MessagePayload.ToJson()
                }.ToJson()
            });

            return(result);
        }
Example #26
0
 /// <summary>
 /// Prepare a message for routing. If we're not in Stop state
 /// the MessageEnqueued event will be raised.
 /// </summary>
 /// <param name="message"></param>
 private void Route(ProduceMessage message)
 {
     if (Post(message))
     {
         MessageEnqueued(message.Topic);
     }
     else
     {
         _cluster.Logger.LogError(
             string.Format(
                 "[Producer] Failed to route message, discarding message for [topic: {0} / partition: {1}]",
                 message.Topic, message.Partition));
         OnMessageDiscarded(message);
     }
 }
Example #27
0
        public void TestProduceTimeout()
        {
            // Prepare
            var connection = new Mock <IConnection>();
            var node       = new Node("Node", () => connection.Object,
                                      new Node.Serialization(null, Compatibility.V0_8_2, Pool, new byte[0], RequiredAcks.Leader, 1, CompressionCodec.None, 0, 100),
                                      new Configuration
            {
                TaskScheduler          = new CurrentThreadTaskScheduler(),
                ErrorStrategy          = ErrorStrategy.Discard,
                ProduceBufferingTime   = TimeSpan.FromMilliseconds(15),
                ProduceBatchSize       = 1,
                ClientRequestTimeoutMs = 2,
                MaxInFlightRequests    = -1
            },
                                      new TimeoutScheduler(3), 1);
            var ackEvent     = new ManualResetEvent(false);
            var timeoutEvent = new ManualResetEvent(false);

            connection.Setup(c => c.SendAsync(It.IsAny <int>(), It.IsAny <ReusableMemoryStream>(), It.IsAny <bool>()))
            .Returns(Success);
            connection.Setup(c => c.ConnectAsync()).Returns(Success);

            node.ProduceAcknowledgement += (n, ack) =>
            {
                Assert.AreSame(node, n);
                Assert.AreEqual(default(ProduceResponse), ack.ProduceResponse);
                Assert.AreNotEqual(default(DateTime), ack.ReceiveDate);
                ackEvent.Set();
            };
            node.RequestTimeout += n =>
            {
                Assert.AreSame(node, n);
                timeoutEvent.Set();
            };

            var exception = new Exception();

            node.ConnectionError += (n, e) =>
            {
                exception = e;
            };

            node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
            WaitHandle.WaitAll(new [] { ackEvent, timeoutEvent });

            Assert.IsInstanceOf <TimeoutException>(exception);
        }
Example #28
0
        public void TestMaxInFlightRequests()
        {
            // Prepare
            var connection = new Mock <IConnection>();
            var node       = new Node("Node", () => connection.Object,
                                      new Node.Serialization(null, Compatibility.V0_8_2, Pool, new byte[0], RequiredAcks.Leader, 1, CompressionCodec.None, 0, 100),
                                      new Configuration
            {
                TaskScheduler          = new CurrentThreadTaskScheduler(),
                ErrorStrategy          = ErrorStrategy.Discard,
                ProduceBufferingTime   = TimeSpan.FromMilliseconds(15),
                ProduceBatchSize       = 1,
                ClientRequestTimeoutMs = 60000,
                MaxInFlightRequests    = 1
            },
                                      new TimeoutScheduler(3), 1);
            var ev    = new ManualResetEvent(false);
            var corrs = new Queue <int>();

            connection.Setup(c => c.SendAsync(It.IsAny <int>(), It.IsAny <ReusableMemoryStream>(), It.IsAny <bool>()))
            .Returns((int c, ReusableMemoryStream d, bool a) =>
            {
                corrs.Enqueue(c);
                return(Success);
            });
            connection.Setup(c => c.ConnectAsync()).Returns(Success);

            node.RequestSent += n =>
            {
                ev.Set();
            };

            node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
            ev.WaitOne();
            ev.Reset();

            node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
            Assert.IsFalse(ev.WaitOne(50));

            var corr = corrs.Dequeue();

            connection.Raise(c => c.Response += null, connection.Object, corr, new ReusableMemoryStream(null));
            node.Produce(ProduceMessage.New("test", 0, new Message(), DateTime.UtcNow.AddDays(1)));
            ev.WaitOne();
        }
Example #29
0
        public void TestUnlimitedRetry()
        {
            SetUp(new Configuration
            {
                MaxRetry = -1
            });

            int discarded = 0;

            _produceRouter.MessageDiscarded += (t, m) => discarded += 1;
            var pm = ProduceMessage.New("test1p", Partitions.Any, new Message(), DateTime.UtcNow.AddMinutes(5));

            _produceRouter.ReEnqueue(pm);
            _produceRouter.ReEnqueue(pm);
            pm.Retried = int.MaxValue;
            _produceRouter.ReEnqueue(pm);
            Assert.AreEqual(0, discarded);
        }
Example #30
0
        /// <summary>
        /// Postpone a message for later check and start the check timer if needed.
        /// The MessagePostponed event is raised.
        /// </summary>
        /// <param name="produceMessage"></param>
        private void PostponeMessage(ProduceMessage produceMessage)
        {
            if (_numberOfPostponedMessages >= _configuration.MaxPostponedMessages)
            {
                _cluster.Logger.LogError(
                    string.Format(
                        "[Producer] Too many postponed messages, discarding message for [topic: {0} / partition: {1}]",
                        produceMessage.Topic, produceMessage.RequiredPartition));
                OnMessageDiscarded(produceMessage);
                return;
            }

            Queue <ProduceMessage> postponedQueue;

            if (!_postponedMessages.TryGetValue(produceMessage.Topic, out postponedQueue))
            {
                postponedQueue = new Queue <ProduceMessage>();
                _postponedMessages.Add(produceMessage.Topic, postponedQueue);
            }
            postponedQueue.Enqueue(produceMessage);
            if (postponedQueue.Count == 1)
            {
                if (produceMessage.RequiredPartition >= 0)
                {
                    _cluster.Logger.LogError(
                        string.Format("[Producer] No node available for [topic: {0} / partition: {1}], postponing messages.",
                                      produceMessage.Topic, produceMessage.RequiredPartition));
                }
                else
                {
                    _cluster.Logger.LogError(string.Format(
                                                 "[Producer] No partition available for topic {0}, postponing messages.", produceMessage.Topic));
                }
            }
            ++_numberOfPostponedMessages;
            MessagePostponed(produceMessage.Topic);

            if (_checkPostponedMessages == null)
            {
                _checkPostponedMessages =
                    new Timer(_ => _messages.Post(RouterMessageType.CheckPostponed),
                              null, _configuration.MessageTtl, _configuration.MessageTtl);
            }
        }