This provider blocks while it attempts to get the MetaData configuration of the Kafka servers. If any retry errors occurs it will continue to block the downstream call and then repeatedly query kafka until the retry errors subside. This repeat call happens in a backoff manner, which each subsequent call waiting longer before a requery. Error Codes: LeaderNotAvailable = 5 NotLeaderForPartition = 6 ConsumerCoordinatorNotAvailableCode = 15 BrokerId = -1 Documentation: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataResponse
Inheritance: IDisposable
        public async Task ShouldBackoffRequestOnMultipleFailures(ErrorResponseCode errorCode)
        {
            var conn = Substitute.For<IKafkaConnection>();

            conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>())
                .Returns(x => CreateMetadataResponse(errorCode),
                            x => CreateMetadataResponse(errorCode),
                            x => CreateMetadataResponse(errorCode),
                            x => CreateMetadataResponse(ErrorResponseCode.NoError));

            using (var provider = new KafkaMetadataProvider(_log))
            {
                var response = await provider.Get(new[] { conn }, new[] { "Test" });
            }

            Received.InOrder(() =>
            {
                conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
                _log.WarnFormat("Backing off metadata request retry.  Waiting for {0}ms.", 100);
                conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
                _log.WarnFormat("Backing off metadata request retry.  Waiting for {0}ms.", 400);
                conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
                _log.WarnFormat("Backing off metadata request retry.  Waiting for {0}ms.", 900);
                conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
            });
        }
        public void NewlyCreatedTopicShouldRetryUntilBrokerIsAssigned()
        {
            var expectedTopic = Guid.NewGuid().ToString();
            var repo = new KafkaMetadataProvider(_options.Log);
            var response = repo.Get(new[] { GetKafkaConnection() }, new[] { expectedTopic });
            var topic = response.Topics.FirstOrDefault();

            Assert.That(topic, Is.Not.Null);
            Assert.That(topic.Name, Is.EqualTo(expectedTopic));
            Assert.That(topic.ErrorCode, Is.EqualTo((int)ErrorResponseCode.NoError));
        }
        public void ShouldReturnWhenNoErrorReceived()
        {
            var conn = Substitute.For<IKafkaConnection>();

            conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>())
                .Returns(x => CreateMetadataResponse(ErrorResponseCode.NoError));

            using (var provider = new KafkaMetadataProvider(_log))
            {
                var response = provider.Get(new[] { conn }, new[] { "Test" });
            }

            conn.Received(1).SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>());
        }
Exemple #4
0
        public BrokerRouter(KafkaOptions kafkaOptions)
        {
            _kafkaOptions = kafkaOptions;
            _kafkaMetadataProvider = new KafkaMetadataProvider(_kafkaOptions.Log);

            foreach (var endpoint in _kafkaOptions.KafkaServerEndpoints)
            {
                var conn = _kafkaOptions.KafkaConnectionFactory.Create(endpoint, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log, _kafkaOptions.MaximumReconnectionTimeout);
                _defaultConnectionIndex.AddOrUpdate(endpoint, e => conn, (e, c) => conn);
            }

            if (_defaultConnectionIndex.Count <= 0)
                throw new ServerUnreachableException("None of the provided Kafka servers are resolvable.");
        }
Exemple #5
0
        /// <exception cref="ServerUnreachableException">None of the provided Kafka servers are resolvable.</exception>

        public BrokerRouter(KafkaOptions kafkaOptions)
        {
            _kafkaOptions          = kafkaOptions;
            _kafkaMetadataProvider = new KafkaMetadataProvider(_kafkaOptions.Log);

            foreach (var endpoint in _kafkaOptions.KafkaServerEndpoints)
            {
                var conn = _kafkaOptions.KafkaConnectionFactory.Create(endpoint, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log, _kafkaOptions.MaxRetry, _kafkaOptions.MaximumReconnectionTimeout, kafkaOptions.StatisticsTrackerOptions);
                _defaultConnectionIndex.AddOrUpdate(endpoint, e => conn, (e, c) => conn);
            }

            if (_defaultConnectionIndex.Count <= 0)
            {
                throw new ServerUnreachableException("None of the provided Kafka servers are resolvable.");
            }
        }
        public async Task ShouldRetryWhenReceiveBrokerIdNegativeOne()
        {
            var conn = Substitute.For<IKafkaConnection>();

            conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>())
                .Returns(x => CreateMetadataResponse(-1, "123", 1), x => CreateMetadataResponse(ErrorResponseCode.NoError));

            using (var provider = new KafkaMetadataProvider(_log))
            {
                var response = await provider.Get(new[] { conn }, new[] { "Test" });
            }

            Received.InOrder(() =>
            {
                conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>());
                _log.WarnFormat("Backing off metadata request retry.  Waiting for {0}ms.", 100);
                conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>());
            });
        }
        public async Task ShouldThrowExceptionWhenNotARetriableErrorCode(ErrorResponseCode errorCode)
        {
            var conn = Substitute.For<IKafkaConnection>();

            conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Returns(x => CreateMetadataResponse(errorCode));

            using (var provider = new KafkaMetadataProvider(_log))
            {
                var response = await provider.Get(new[] { conn }, new[] { "Test" });
            }
        }
        public async Task ShouldThrowExceptionWhenPortIsMissing(int port)
        {
            var conn = Substitute.For<IKafkaConnection>();

            conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Returns(x => CreateMetadataResponse(1, "123", port));

            using (var provider = new KafkaMetadataProvider(_log))
            {
                var response = await provider.Get(new[] { conn }, new[] { "Test" });
            }
        }
        public void ShouldThrowExceptionWhenHostIsMissing(string host)
        {
            var conn = Substitute.For<IKafkaConnection>();

            conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Returns(x => CreateMetadataResponse(1, host, 1));

            using (var provider = new KafkaMetadataProvider(_log))
            {
                var response = provider.Get(new[] { conn }, new[] { "Test" });
            }
        }