public async Task UpdateOrCreateOffsetConsumerGroupExistsTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config); var partitionId = 0; var consumerGroup = TestConfig.ConsumerName(); var topicName = TestConfig.TopicName(); var offest = 5; var newOffset = 10; await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, CancellationToken.None); await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, offest, CancellationToken.None); var res = await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None); Assert.AreEqual(offest, res.Offset); await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, newOffset, CancellationToken.None); res = await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None); Assert.AreEqual(newOffset, res.Offset); }
public async Task FetchMessagesCacheContainsAllRequestTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_options); var producer = new Producer(brokerRouter); var topic = TestConfig.TopicName(); var consumer = new Consumer(brokerRouter, _consumerConfig); var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None); // Creating 5 messages var messages = CreateTestMessages(10, 1); await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None); // Now let's consume var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList(); CheckMessages(messages.Take(5).ToList(), result); // Now let's consume again result = (await consumer.FetchMessagesAsync(offset.TopicName, offset.PartitionId, offset.Offset + 5, 5, CancellationToken.None)).ToList(); CheckMessages(messages.Skip(5).ToList(), result); }
public async Task FetchMessagesBufferUnderRunTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_options); var smallMessageSet = 4096 / 2; var producer = new Producer(brokerRouter); var topic = TestConfig.TopicName(); var consumer = new Consumer(brokerRouter, new ConsumerConfiguration(maxPartitionFetchBytes: smallMessageSet)); var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None); // Creating 5 messages var messages = CreateTestMessages(10, 4096); await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None); try { // Now let's consume await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None); Assert.Fail("should have thrown BufferUnderRunException"); } catch (BufferUnderRunException ex) { Console.WriteLine(ex.ToString()); } }
public async Task ProducerShouldUsePartitionIdInsteadOfMessageKeyToChoosePartition() { var partitionSelector = Substitute.For <IPartitionSelector>(); partitionSelector.Select(null, null) .ReturnsForAnyArgs(_ => _.Arg <MetadataResponse.Topic>().Partitions.Single(p => p.PartitionId == 1)); using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri, partitionSelector: partitionSelector))) { var offset = await router.GetTopicOffsetAsync(TestConfig.TopicName(), 0, CancellationToken.None); using (var producer = new Producer(router)) { //message should send to PartitionId and not use the key to Select Broker Route !! for (var i = 0; i < 20; i++) { await producer.SendMessageAsync(new Message(i.ToString(), "key"), offset.TopicName, offset.PartitionId, CancellationToken.None); } } using (var consumer = new Consumer(router)) { using (var source = new CancellationTokenSource()) { var i = 0; await consumer.FetchAsync( offset, 20, message => { Assert.That(message.Value.ToUtf8String(), Is.EqualTo(i++.ToString())); if (i >= 20) { source.Cancel(); } return(Task.FromResult(0)); }, source.Token); } } } }
public async Task FetchLastOffsetTopicDoesntExistTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config, log: TestConfig.InfoLog); var topic = TestConfig.TopicName(); await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None); }
public async Task FetchLastOffsetPartitionDoesntExistTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config); var partitionId = 100; var topic = TestConfig.TopicName(); Assert.ThrowsAsync <CachedMetadataException>(async() => await brokerRouter.GetTopicOffsetAsync(topic, partitionId, CancellationToken.None)); }
public async Task FetchOffsetConsumerGroupDoesntExistTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config); var partitionId = 0; var consumerGroup = Guid.NewGuid().ToString(); var topicName = TestConfig.TopicName(); await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None); }
public async Task FetchLastOffsetSimpleTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config); var topic = TestConfig.TopicName(); var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None); Assert.AreNotEqual(-1, offset.Offset); }
public async Task FetchMessagesNoNewMessagesInQueueTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config); var consumer = new Consumer(brokerRouter, _consumerConfig); var offset = await brokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), _partitionId, CancellationToken.None); // Now let's consume var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList(); Assert.AreEqual(0, result.Count, "Should not get any messages"); }
public async Task ManualConsumerFailure() { var topicName = "TestTopicIssue13-3R-1P"; using (var router = new BrokerRouter(_options)) { var consumer = new Consumer(new BrokerRouter(_options), new ConsumerConfiguration(maxPartitionFetchBytes: 10000)); var offset = await router.GetTopicOffsetAsync(topicName, 0, CancellationToken.None); var producer = new Producer(router); var send = SandMessageForever(producer, offset.TopicName, offset.PartitionId); var read = ReadMessageForever(consumer, offset.TopicName, offset.PartitionId, offset.Offset); await Task.WhenAll(send, read); } }
public async Task FetchOffsetConsumerGroupArgumentNull([Values(null, "")] string group) { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config); var partitionId = 0; var consumerGroup = TestConfig.ConsumerName(); var topicName = TestConfig.TopicName(); var offset = 5; await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, offset, CancellationToken.None); Assert.ThrowsAsync <ArgumentNullException>(async() => await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, group, CancellationToken.None)); }
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config, log: TestConfig.InfoLog); var consumer = new Consumer(brokerRouter, _consumerConfig); var offset = await brokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), _partitionId, CancellationToken.None); try { // Now let's consume await consumer.FetchMessagesAsync(offset.TopicName, offset.PartitionId, offset.Offset + 1, 5, CancellationToken.None); Assert.Fail("should have thrown FetchOutOfRangeException"); } catch (FetchOutOfRangeException ex) when(ex.Message.StartsWith("Kafka returned OffsetOutOfRange for Fetch request")) { Console.WriteLine(ex.ToString()); } }
public async Task FetchMessagesSimpleTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_options); var topic = "ManualConsumerTestTopic"; var producer = new Producer(brokerRouter); var consumer = new Consumer(brokerRouter, _consumerConfig); var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None); // Creating 5 messages var messages = CreateTestMessages(5, 1); await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None); // Now let's consume var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList(); CheckMessages(messages, result); }