Exemple #1
0
        public async Task ShouldUpdateMetadataOnes()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var             router          = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction    = ShouldReturnValidMessage;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            int numberOfCall = 1000;

            Task[] tasks = new Task[numberOfCall];
            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
            }
            await Task.Delay(routerProxy._cacheExpiration);

            await Task.Delay(1);

            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
            }

            await Task.WhenAll(tasks);

            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
        }
Exemple #2
0
        public async Task FetchMessagesCacheContainsNoneOfRequestTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter    = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            Producer       producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List <Message> messages = CreateTestMessages(10, 4096);

            await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(7, offset)).ToList();

            CheckMessages(messages.Take(7).ToList(), result);

            // Now let's consume again
            result = (await consumer.FetchMessages(2, offset + 8)).ToList();

            CheckMessages(messages.Skip(8).ToList(), result);
        }
        public async Task FetchMessagesCacheContainsAllRequestTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            Producer producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List<Message> messages = CreateTestMessages(10, 1);

            await producer.SendMessageAsync(_topic, messages, partition: _partitionId, timeout: TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            CheckMessages(messages.Take(5).ToList(), result);

            // Now let's consume again
            result = (await consumer.FetchMessages(5, offset + 5)).ToList();

            CheckMessages(messages.Skip(5).ToList(), result);
        }
        public async Task ProtocolGateway()
        {
            int partitionId = 0;
            var router = new BrokerRouter(Options);

            var producer = new Producer(router);
            string messge1 = Guid.NewGuid().ToString();
            var respose = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(messge1) }, 1, null, MessageCodec.CodecNone, partitionId);
            var offset = respose.FirstOrDefault().Offset;

            ProtocolGateway protocolGateway = new ProtocolGateway(IntegrationConfig.IntegrationUri);
            var fetch = new Fetch
                         {
                             Topic = IntegrationConfig.IntegrationTopic,
                             PartitionId = partitionId,
                             Offset = offset,
                             MaxBytes = 32000,
                         };

            var fetches = new List<Fetch> { fetch };

            var fetchRequest = new FetchRequest
                {
                    MaxWaitTime = 1000,
                    MinBytes = 10,
                    Fetches = fetches
                };

            var r = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
            //  var r1 = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
            Assert.IsTrue(r.Messages.FirstOrDefault().Value.ToUtf8String() == messge1);
        }
        public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = new TimeSpan(10);
            var router = routerProxy.Create();

            int partitionId = 0;

            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var fetchRequest = new FetchRequest();
            bool sendExOnFirstTime = true;

            Func<Task<FetchResponse>> ShouldReturnErrorAndThenNoError = async () =>
            {
                Task.Delay(routerProxy._cacheExpiration).Wait();
                Task.Delay(1).Wait();
                if (sendExOnFirstTime)
                {
                    sendExOnFirstTime = false;
                    return new FetchResponse() { Error = (short)code };
                }
                return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
            };
            routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnErrorAndThenNoError;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;

            await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);

            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
        }
Exemple #6
0
        public async Task ShouldRecoverFromFailerByUpdateMetadataOnce() //Do not debug this test !!
        {
            var log         = new DefaultTraceLog();
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(1000);
            var router = routerProxy.Create();

            int             partitionId     = 0;
            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var             fetchRequest    = new FetchRequest();

            int  numberOfCall              = 100;
            long numberOfErrorSend         = 0;
            TaskCompletionSource <int>   x = new TaskCompletionSource <int>();
            Func <Task <FetchResponse> > ShouldReturnNotLeaderForPartitionAndThenNoError = async() =>
            {
                log.DebugFormat("FetchResponse Start ");
                if (!x.Task.IsCompleted)
                {
                    if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall)
                    {
                        await Task.Delay(routerProxy._cacheExpiration);

                        await Task.Delay(1);

                        x.TrySetResult(1);
                        log.DebugFormat("all is complete ");
                    }

                    await x.Task;
                    log.DebugFormat("SocketException ");
                    throw new BrokerConnectionException("", new KafkaEndpoint());
                }
                log.DebugFormat("Completed ");

                return(new FetchResponse()
                {
                    Error = (short)ErrorResponseCode.NoError
                });
            };

            routerProxy.BrokerConn0.FetchResponseFunction    = ShouldReturnNotLeaderForPartitionAndThenNoError;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;

            Task[] tasks = new Task[numberOfCall];

            for (int i = 0; i < numberOfCall; i++)
            {
                tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
            }

            await Task.WhenAll(tasks);

            Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend");
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend),
                        "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount");
        }
Exemple #7
0
 public async Task ShouldThrowFormatExceptionWhenTopicIsInvalid()
 {
     var             routerProxy     = new BrokerRouterProxy(_kernel);
     var             router          = routerProxy.Create();
     string          invalidTopic    = " ";
     var             fetchRequest    = new FetchRequest();
     ProtocolGateway protocolGateway = new ProtocolGateway(router);
     await protocolGateway.SendProtocolRequest(fetchRequest, invalidTopic, 0);
 }
 public async Task ShouldThrowFormatExceptionWhenTopicIsInvalid()
 {
     var routerProxy = new BrokerRouterProxy(_kernel);
     var router = routerProxy.Create();
     string invalidTopic = " ";
     var fetchRequest = new FetchRequest();
     ProtocolGateway protocolGateway = new ProtocolGateway(router);
     await protocolGateway.SendProtocolRequest(fetchRequest, invalidTopic, 0);
 }
Exemple #9
0
        public async Task FetchOffsetConsumerGroupDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId     = 0;
            var consumerGroup   = Guid.NewGuid().ToString();

            ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            await consumer.FetchOffset(consumerGroup);
        }
Exemple #10
0
        public async Task FetchLastOffsetPartitionDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId     = 100;
            var topic           = _topic;

            ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            await consumer.FetchLastOffset();
        }
Exemple #11
0
        public async Task FetchMessagesInvalidOffsetTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = -1;

            // Now let's consume
            await consumer.FetchMessages(5, offset);
        }
Exemple #12
0
        public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Now let's consume throw KafkaApplicationException
            await consumer.FetchMessages(5, offset + 1);
        }
Exemple #13
0
        public async Task UpdateOrCreateOffsetConsumerGroupEmptyTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId     = 0;
            var topic           = _topic + Guid.NewGuid();

            ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offest = 5;

            await consumer.UpdateOrCreateOffset(string.Empty, offest);
        }
Exemple #14
0
        public async Task FetchLastOffsetTopicDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = _topic + Guid.NewGuid();

            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            var res = await consumer.FetchLastOffset();

            Assert.AreEqual(0, res);
        }
Exemple #15
0
        public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(
            ErrorResponseCode code)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var             router          = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction    = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
        }
        public ManualConsumer(int partitionId, string topic, ProtocolGateway gateway, string clientId, int maxSizeOfMessageSet)
        {
            if (string.IsNullOrEmpty(topic)) throw new ArgumentNullException("topic");
            if (gateway == null) throw new ArgumentNullException("gateway");
            if (maxSizeOfMessageSet <= 0) throw new ArgumentOutOfRangeException("maxSizeOfMessageSet", "argument must be larger than zero");

            _gateway = gateway;
            _partitionId = partitionId;
            _topic = topic;
            _clientId = clientId;

            _maxSizeOfMessageSet = maxSizeOfMessageSet;
        }
Exemple #17
0
        public async Task FetchLastOffsetSimpleTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = _topic;

            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            var offset = await consumer.FetchLastOffset();

            Assert.AreNotEqual(-1, offset);
        }
Exemple #18
0
        public async Task FetchMessagesNoNewMessagesInQueueTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            Assert.AreEqual(0, result.Count, "Should not get any messages");
        }
Exemple #19
0
        public async Task FetchOffsetConsumerGroupIsNullTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId     = 0;
            var consumerGroup   = _defaultConsumerGroup;

            ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offest = 5;

            await consumer.UpdateOrCreateOffset(consumerGroup, offest);

            await consumer.FetchOffset(null);
        }
        public async Task ShouldTryToRefreshMataDataIfOnExceptions(Type exceptionType)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;

            await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);

            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
        }
        public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = new TimeSpan(10);
            var router = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;

            await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);

            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
        }
Exemple #22
0
        public async Task ShouldTryToRefreshMataDataIfCanRecoverByRefreshMetadata(ErrorResponseCode code)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = new TimeSpan(10);
            var             router          = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction    = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;

            await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);

            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
        }
Exemple #23
0
        public async Task ShouldTryToRefreshMataDataIfSocketException()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var             router          = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction    = FailedInFirstMessageException(typeof(BrokerConnectionException), routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;

            await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);

            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(2));
        }
        public KafkaQueueAdapter(HashRingBasedStreamQueueMapper queueMapper, KafkaStreamProviderOptions options,
                                 string providerName, IKafkaBatchFactory batchFactory, ILogger logger)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (batchFactory == null)
            {
                throw new ArgumentNullException(nameof(batchFactory));
            }
            if (queueMapper == null)
            {
                throw new ArgumentNullException(nameof(queueMapper));
            }
            if (logger == null)
            {
                throw new ArgumentNullException(nameof(logger));
            }
            if (string.IsNullOrEmpty(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }

            _options           = options;
            _streamQueueMapper = queueMapper;
            Name          = providerName;
            _batchFactory = batchFactory;
            _logger       = logger;

            // Creating a producer
            KafkaOptions kafkaOptions = new KafkaOptions(_options.ConnectionStrings.ToArray())
            {
                Log = new KafkaLogBridge(logger)
            };
            var broker = new BrokerRouter(kafkaOptions);

            _producer = new Producer(broker)
            {
                BatchDelayTime = TimeSpan.FromMilliseconds(_options.TimeToWaitForBatchInMs),
                BatchSize      = _options.ProduceBatchSize
            };
            _gateway = new ProtocolGateway(kafkaOptions);

            _logger.Info("KafkaQueueAdapter - Created a KafkaQueueAdapter");
        }
        /// <summary>
        /// Construct a Producer class.
        /// </summary>
        /// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
        /// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests.  -1 indicates unlimited.</param>
        /// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
        /// <remarks>
        /// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
        /// by blocking the caller requesting the async call.  This affectively puts an upper limit on the amount of times a caller can
        /// call SendMessageAsync before the caller is blocked.
        ///
        /// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
        /// overwhelmed and the buffer starts to fill up.  This is an inaccurate limiting memory use as the amount of memory actually used is
        /// dependant on the general message size being buffered.
        ///
        /// A message will start its timeout countdown as soon as it is added to the producer async queue.  If there are a large number of
        /// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
        /// attempted to send to Kafka before a timeout exception is thrown.
        /// </remarks>
        public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
        {
            BrokerRouter = brokerRouter;
            _protocolGateway = new ProtocolGateway(BrokerRouter);
            _maximumAsyncRequests = maximumAsyncRequests;
            _metadataQueries = new MetadataQueries(BrokerRouter);
            _asyncCollection = new AsyncCollection<TopicMessage>();
            _semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);

            BatchSize = DefaultBatchSize;
            BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);

            _postTask = Task.Run(() =>
            {
                BatchSendAsync();
                BrokerRouter.Log.InfoFormat("ending the sending thread");
            });
        }
Exemple #26
0
        public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var             router          = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction    = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            try
            {
                await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);

                Assert.IsTrue(false, "Should throw exception");
            }
            catch (Exception ex)
            {
                Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
            }
        }
Exemple #27
0
        public async Task ShouldRecoverFromFailerByUpdateMetadataOnceFullScenario1()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(0);
            var             router          = routerProxy.Create();
            int             partitionId     = 0;
            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var             fetchRequest    = new FetchRequest();

            CreateSuccessfulSendMock(routerProxy);

            //Send Successful Message
            await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);

            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
            Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");

            routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(ErrorResponseCode.LeaderNotAvailable, TimeSpan.Zero);

            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
            routerProxy.BrokerConn1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;

            //Reset variables
            routerProxy.BrokerConn0.FetchRequestCallCount    = 0;
            routerProxy.BrokerConn1.FetchRequestCallCount    = 0;
            routerProxy.BrokerConn0.MetadataRequestCallCount = 0;
            routerProxy.BrokerConn1.MetadataRequestCallCount = 0;

            //Send Successful Message that was recover from exception
            await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);

            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");

            Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
        }
Exemple #28
0
        public async Task ShouldRecoverUpdateMetadataForNewTopic()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);

            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();

            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var             fetchRequest    = new FetchRequest();

            routerProxy.BrokerConn0.FetchResponseFunction    = ShouldReturnValidMessage;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            int numberOfCall = 1000;

            Task[] tasks = new Task[numberOfCall];
            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, _partitionId);
            }

            routerProxy.BrokerConn0.MetadataResponseFunction = async() =>
            {
                var response = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();

                response.Topics[0].Name = "test2";
                return(response);
            };

            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", _partitionId);
            }

            await Task.WhenAll(tasks);

            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
        }
        public async Task FetchMessagesSimpleTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId = 1;
            var topic = "ManualConsumerTestTopic";

            Producer producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List<Message> messages = CreateTestMessages(5, 1);

            await producer.SendMessageAsync(topic, messages, partition: partitionId, timeout: TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            CheckMessages(messages, result);
        }
Exemple #30
0
        public async Task FetchMessagesBufferUnderRunTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter    = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var smallMessageSet = 4096 / 2;

            Producer       producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List <Message> messages = CreateTestMessages(10, 4096);

            await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            CheckMessages(messages.Take(5).ToList(), result);
        }
Exemple #31
0
        public async Task FetchMessagesSimpleTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter    = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = "ManualConsumerTestTopic";

            Producer       producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List <Message> messages = CreateTestMessages(5, 1);

            await producer.SendMessageAsync(topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            CheckMessages(messages, result);
        }
Exemple #32
0
        public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId     = 0;
            var consumerGroup   = _defaultConsumerGroup + Guid.NewGuid();

            ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offest    = 5;
            var newOffset = 10;

            await consumer.UpdateOrCreateOffset(consumerGroup, offest);

            var res = await consumer.FetchOffset(consumerGroup);

            Assert.AreEqual(offest, res);

            await consumer.UpdateOrCreateOffset(consumerGroup, newOffset);

            res = await consumer.FetchOffset(consumerGroup);

            Assert.AreEqual(newOffset, res);
        }
Exemple #33
0
        public async Task ProtocolGateway()
        {
            int partitionId = 0;
            var router      = new BrokerRouter(Options);

            var    producer = new Producer(router);
            string messge1  = Guid.NewGuid().ToString();
            var    respose  = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(messge1) }, 1, null, MessageCodec.CodecNone, partitionId);

            var offset = respose.FirstOrDefault().Offset;

            ProtocolGateway protocolGateway = new ProtocolGateway(IntegrationConfig.IntegrationUri);
            var             fetch           = new Fetch
            {
                Topic       = IntegrationConfig.IntegrationTopic,
                PartitionId = partitionId,
                Offset      = offset,
                MaxBytes    = 32000,
            };

            var fetches = new List <Fetch> {
                fetch
            };

            var fetchRequest = new FetchRequest
            {
                MaxWaitTime = 1000,
                MinBytes    = 10,
                Fetches     = fetches
            };

            var r = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);

            //  var r1 = await protocolGateway.SendProtocolRequest(fetchRequest, IntegrationConfig.IntegrationTopic, partitionId);
            Assert.IsTrue(r.Messages.FirstOrDefault().Value.ToUtf8String() == messge1);
        }
        public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId = 0;
            var consumerGroup = _defaultConsumerGroup + Guid.NewGuid();

            ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offest = 5;
            var newOffset = 10;

            await consumer.UpdateOrCreateOffset(consumerGroup, offest);
            var res = await consumer.FetchOffset(consumerGroup);
            Assert.AreEqual(offest, res);

            await consumer.UpdateOrCreateOffset(consumerGroup, newOffset);

            res = await consumer.FetchOffset(consumerGroup);

            Assert.AreEqual(newOffset, res);
        }
        public async Task UpdateOrCreateOffsetPartitionDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId = 100;
            var consumerGroup = Guid.NewGuid().ToString();

            ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offest = 5;

            await consumer.UpdateOrCreateOffset(consumerGroup, offest);
        }
        public async Task FetchOffsetTopicDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = _topic + Guid.NewGuid();
            var consumerGroup = _defaultConsumerGroup;

            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            await consumer.FetchOffset(consumerGroup);
        }
        public async Task FetchOffsetConsumerGroupIsEmptyTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId = 0;
            var consumerGroup = _defaultConsumerGroup;

            ManualConsumer consumer = new ManualConsumer(partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offest = 5;

            await consumer.UpdateOrCreateOffset(consumerGroup, offest);
            await consumer.FetchOffset(string.Empty);
        }
        public async Task FetchMessagesTopicDoesntExist()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId = 1;
            var topic = _topic + Guid.NewGuid();

            ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            var offset = await consumer.FetchLastOffset();

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            Assert.AreEqual(0, result.Count);
        }
        public async Task ShouldRecoverFromFailerByUpdateMetadataOnce() //Do not debug this test !!
        {
            var log = new DefaultTraceLog();
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(1000);
            var router = routerProxy.Create();

            int partitionId = 0;
            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var fetchRequest = new FetchRequest();

            int numberOfCall = 100;
            long numberOfErrorSend = 0;
            TaskCompletionSource<int> x = new TaskCompletionSource<int>();
            Func<Task<FetchResponse>> ShouldReturnNotLeaderForPartitionAndThenNoError = async () =>
            {
                log.DebugFormat("FetchResponse Start ");
                if (!x.Task.IsCompleted)
                {
                    if (Interlocked.Increment(ref numberOfErrorSend) == numberOfCall)
                    {
                        await Task.Delay(routerProxy._cacheExpiration);
                        await Task.Delay(1);
                        x.TrySetResult(1);
                        log.DebugFormat("all is complete ");
                    }

                    await x.Task;
                    log.DebugFormat("SocketException ");
                    throw new BrokerConnectionException("",new KafkaEndpoint());
                }
                log.DebugFormat("Completed ");

                return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
            };

            routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnNotLeaderForPartitionAndThenNoError;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;

            Task[] tasks = new Task[numberOfCall];

            for (int i = 0; i < numberOfCall; i++)
            {
                tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
            }

            await Task.WhenAll(tasks);
            Assert.That(numberOfErrorSend, Is.GreaterThan(1), "numberOfErrorSend");
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall + numberOfErrorSend),
                "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2), "MetadataRequestCallCount");
        }
        public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(ErrorResponseCode code)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();

            int partitionId = 0;
            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var fetchRequest = new FetchRequest();

            bool firstTime = true;
            Func<Task<FetchResponse>> ShouldReturnError = async () =>
            {
                if (firstTime)
                {
                    firstTime = !firstTime;
                    return new FetchResponse() { Error = (short)code };
                }
                return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
            };
            routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
            await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
        }
        public async Task FetchLastOffsetPartitionDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId = 100;
            var topic = _topic;

            ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            await consumer.FetchLastOffset();
        }
        public async Task ShouldRecoverUpdateMetadataForNewTopic()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();

            int partitionId = 0;
            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var fetchRequest = new FetchRequest();

            bool firstTime = true;
            Func<Task<FetchResponse>> ShouldReturnError = async () =>
             {
                 return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
             };

            routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
            int numberOfCall = 1000;
            Task[] tasks = new Task[numberOfCall];
            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
            }
            //change MetadataResponseFunction
            routerProxy.BrokerConn0.MetadataResponseFunction = async () =>
            {
                var response = await BrokerRouterProxy.DefaultMetadataResponse();
                response.Topics[0].Name = "test2";
                return response;
            };

            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", partitionId);
            }

            await Task.WhenAll(tasks);
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
        }
        public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageException(exceptionType, routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            try
            {
                await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
                Assert.IsTrue(false, "Should throw exception");
            }
            catch (Exception ex)
            {
                Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
            }
        }
        public async Task SendProtocolRequestShouldNoTryToRefreshMataDataIfCanNotRecoverByRefreshMetadata(
            ErrorResponseCode code)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(code, routerProxy._cacheExpiration);
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            await protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
        }
        public async Task ShouldUpdateMetadataOnes()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();
            ProtocolGateway protocolGateway = new ProtocolGateway(router);

            routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            int numberOfCall = 1000;
            Task[] tasks = new Task[numberOfCall];
            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
            }
            await Task.Delay(routerProxy._cacheExpiration);
            await Task.Delay(1);
            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(new FetchRequest(), BrokerRouterProxy.TestTopic, _partitionId);
            }

            await Task.WhenAll(tasks);
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1));
        }
        public async Task ShouldRecoverFromFailerByUpdateMetadataOnceFullScenario1()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(0);
            var router = routerProxy.Create();
            int partitionId = 0;
            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var fetchRequest = new FetchRequest();

            CreateSuccessfulSendMock(routerProxy);

            //Send Successful Message
            await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);

            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");
            Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");

            routerProxy.BrokerConn0.FetchResponseFunction = FailedInFirstMessageError(ErrorResponseCode.LeaderNotAvailable, TimeSpan.Zero);

            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;
            routerProxy.BrokerConn1.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithSingleBroker;

            //Reset variables
            routerProxy.BrokerConn0.FetchRequestCallCount = 0;
            routerProxy.BrokerConn1.FetchRequestCallCount = 0;
            routerProxy.BrokerConn0.MetadataRequestCallCount = 0;
            routerProxy.BrokerConn1.MetadataRequestCallCount = 0;

            //Send Successful Message that was recover from exception
            await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);

            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(1), "MetadataRequestCallCount");

            Assert.That(routerProxy.BrokerConn1.FetchRequestCallCount, Is.EqualTo(1), "FetchRequestCallCount");
            Assert.That(routerProxy.BrokerConn1.MetadataRequestCallCount, Is.EqualTo(0), "MetadataRequestCallCount");
        }
        public async Task UpdateOrCreateOffsetNegativeOffsetTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId = 0;
            var topic = _topic + Guid.NewGuid();
            var consumerGroup = _defaultConsumerGroup;

            ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offest = -5;

            await consumer.UpdateOrCreateOffset(consumerGroup, offest);
        }
        public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset + 1)).ToList();

            Assert.AreEqual(0, result.Count, "Should not get any messages");
        }
        public async Task FetchLastOffsetSimpleTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = _topic;

            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            var offset = await consumer.FetchLastOffset();

            Assert.AreNotEqual(-1, offset);
        }
        public async Task FetchMessagesInvalidOffsetTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = -1;

            // Now let's consume
            await consumer.FetchMessages(5, offset);
        }
        public async Task FetchLastOffsetTopicDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = _topic + Guid.NewGuid();

            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            var res = await consumer.FetchLastOffset();

            Assert.AreEqual(0, res);
        }
        public async Task FetchMessagesBufferUnderRunTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var smallMessageSet = 4096 / 2;

            Producer producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List<Message> messages = CreateTestMessages(10, 4096);

            await producer.SendMessageAsync(_topic, messages, partition: _partitionId, timeout: TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            CheckMessages(messages.Take(5).ToList(), result);
        }
        public async Task SendProtocolRequestShouldThrowException(Type exceptionType)
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();

            int partitionId = 0;
            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var fetchRequest = new FetchRequest();

            bool firstTime = true;
            Func<Task<FetchResponse>> ShouldReturnError = async () =>
            {
                if (firstTime)
                {
                    firstTime = !firstTime;
                    object[] args = new object[1];
                    args[0] = "error Test";
                    throw (Exception)Activator.CreateInstance(exceptionType, args);
                }
                return new FetchResponse() { Error = (short)ErrorResponseCode.NoError };
            };

            routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnError;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.DefaultMetadataResponse;
            try
            {
                await protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, partitionId);
                Assert.IsTrue(false, "Should throw exception");
            }
            catch (Exception ex)
            {
                Assert.That(ex.GetType(), Is.EqualTo(exceptionType));
            }
        }
        public async Task ShouldRecoverUpdateMetadataForNewTopic()
        {
            var routerProxy = new BrokerRouterProxy(_kernel);
            routerProxy._cacheExpiration = TimeSpan.FromMilliseconds(10);
            var router = routerProxy.Create();

            ProtocolGateway protocolGateway = new ProtocolGateway(router);
            var fetchRequest = new FetchRequest();

            routerProxy.BrokerConn0.FetchResponseFunction = ShouldReturnValidMessage;
            routerProxy.BrokerConn0.MetadataResponseFunction = BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers;
            int numberOfCall = 1000;
            Task[] tasks = new Task[numberOfCall];
            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i] = protocolGateway.SendProtocolRequest(fetchRequest, BrokerRouterProxy.TestTopic, _partitionId);
            }

            routerProxy.BrokerConn0.MetadataResponseFunction = async () =>
            {
                var response = await BrokerRouterProxy.CreateMetadataResponseWithMultipleBrokers();
                response.Topics[0].Name = "test2";
                return response;
            };

            for (int i = 0; i < numberOfCall / 2; i++)
            {
                tasks[i + numberOfCall / 2] = protocolGateway.SendProtocolRequest(fetchRequest, "test2", _partitionId);
            }

            await Task.WhenAll(tasks);
            Assert.That(routerProxy.BrokerConn0.FetchRequestCallCount, Is.EqualTo(numberOfCall));
            Assert.That(routerProxy.BrokerConn0.MetadataRequestCallCount, Is.EqualTo(2));
        }