public async Task CanFetch() { int partitionId = 0; var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri)); var producer = new Producer(router); string messageValue = Guid.NewGuid().ToString(); var response = await producer.SendMessageAsync(new Message(messageValue), TestConfig.TopicName(), partitionId, CancellationToken.None); var offset = response.Offset; var fetch = new FetchRequest.Topic(TestConfig.TopicName(), partitionId, offset, 32000); var fetchRequest = new FetchRequest(fetch, minBytes: 10); var r = await router.SendAsync(fetchRequest, TestConfig.TopicName(), partitionId, CancellationToken.None); Assert.IsTrue(r.Topics.First().Messages.First().Value.ToUtf8String() == messageValue); }
public async Task SendAsyncShouldHandleHighVolumeOfMessages(int amount, int maxAsync) { using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri))) using (var producer = new Producer(router, new ProducerConfiguration(maxAsync, amount / 2))) { var tasks = new Task <ProduceResponse.Topic> [amount]; for (var i = 0; i < amount; i++) { tasks[i] = producer.SendMessageAsync(new Message(Guid.NewGuid().ToString()), TestConfig.TopicName(), CancellationToken.None); } var results = await Task.WhenAll(tasks.ToArray()); //Because of how responses are batched up and sent to servers, we will usually get multiple responses per requested message batch //So this assertion will never pass //Assert.That(results.Count, Is.EqualTo(amount)); Assert.That(results.Any(x => x.ErrorCode != ErrorResponseCode.None), Is.False, "Should not have received any results as failures."); } }
public async Task FetchMessagesInvalidOffsetTest() { // Creating a broker router and a protocol gateway for the producer and consumer var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config); var consumer = new Consumer(brokerRouter, _consumerConfig); var offset = -1; // Now let's consume Assert.ThrowsAsync <ArgumentOutOfRangeException>(async() => await consumer.FetchMessagesAsync(TestConfig.TopicName(), _partitionId, offset, 5, CancellationToken.None)); }
public async Task ConsumeByOffsetShouldGetSameMessageProducedAtSameOffset() { long offsetResponse; var messge = Guid.NewGuid(); using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) { using (var producer = new Producer(router)) { var responseAckLevel1 = await producer.SendMessageAsync(new Message(messge.ToString()), TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 1), CancellationToken.None); offsetResponse = responseAckLevel1.Offset; } } using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) { using (var consumer = new Consumer(router, new ConsumerConfiguration(maxServerWait: TimeSpan.Zero))) { var result = await consumer.FetchMessagesAsync(TestConfig.TopicName(), 0, offsetResponse, 1, CancellationToken.None); Assert.AreEqual(messge.ToString(), result[0].Value.ToUtf8String()); } } }
public async Task ConsumerShouldConsumeInSameOrderAsProduced([Values(20)] int sends, [Values(1, 10)] int messagesPerSend) { var totalMessages = sends * messagesPerSend; var expected = totalMessages.Repeat(i => i.ToString()).ToList(); var testId = Guid.NewGuid().ToString(); using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) { using (var producer = new Producer(router)) { var offset = await producer.BrokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), 0, CancellationToken.None); for (var i = 0; i < sends; i++) { if (messagesPerSend == 1) { await producer.SendMessageAsync(new Message(i.ToString(), testId), TestConfig.TopicName(), 0, CancellationToken.None); } else { var current = i * messagesPerSend; var messages = messagesPerSend.Repeat(_ => new Message((current + _).ToString(), testId)).ToList(); await producer.SendMessagesAsync(messages, TestConfig.TopicName(), 0, CancellationToken.None); } } using (var consumer = new Consumer(router, new ConsumerConfiguration(maxServerWait: TimeSpan.Zero))) { var results = await consumer.FetchMessagesAsync(offset, totalMessages, CancellationToken.None); //ensure the produced messages arrived TestConfig.InfoLog.Info(() => LogEvent.Create($"Message order: {string.Join(", ", results.Select(x => x.Value.ToUtf8String()).ToList())}")); Assert.That(results.Count, Is.EqualTo(totalMessages)); Assert.That(results.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected), "Expected the message list in the correct order."); Assert.That(results.Any(x => x.Key.ToUtf8String() != testId), Is.False); } } } }
public async Task ProducerLastResposeOffsetAckLevel1ShouldBeEqualsToLastOffset() { using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) using (var producer = new Producer(router)) { var responseAckLevel1 = await producer.SendMessagesAsync(new [] { new Message("Ack Level 1"), new Message("Ack Level 1") }, TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 1), CancellationToken.None); var offsetResponse = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None); var maxOffset = offsetResponse.First(x => x.PartitionId == 0); Assert.AreEqual(responseAckLevel1.Last().Offset, maxOffset.Offset - 1); } }
public async Task ConsumerShouldMoveToNextAvailableOffsetWhenQueryingForNextMessage() { const int expectedCount = 1000; var options = new KafkaOptions(TestConfig.IntegrationUri); using (var producerRouter = new BrokerRouter(options)) using (var producer = new Producer(producerRouter)) { //get current offset and reset consumer to top of log var offsets = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).ConfigureAwait(false); using (var consumerRouter = new BrokerRouter(options)) using (var consumer = new OldConsumer(new ConsumerOptions(TestConfig.TopicName(), consumerRouter) { MaxWaitTimeForMinimumBytes = TimeSpan.Zero }, offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offset)).ToArray())) { Console.WriteLine("Sending {0} test messages", expectedCount); var response = await producer.SendMessagesAsync(Enumerable.Range(0, expectedCount).Select(x => new Message(x.ToString())), TestConfig.TopicName(), CancellationToken.None); Assert.That(response.Any(x => x.ErrorCode != (int)ErrorResponseCode.None), Is.False, "Error occured sending test messages to server."); var stream = consumer.Consume(); Console.WriteLine("Reading message back out from consumer."); var data = stream.Take(expectedCount).ToList(); var consumerOffset = consumer.GetOffsetPosition().OrderBy(x => x.PartitionId).ToList(); var serverOffset = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).ConfigureAwait(false); var positionOffset = serverOffset.Select(x => new OffsetPosition(x.PartitionId, x.Offset)) .OrderBy(x => x.PartitionId) .ToList(); Assert.That(consumerOffset, Is.EqualTo(positionOffset), "The consumerOffset position should match the server offset position."); Assert.That(data.Count, Is.EqualTo(expectedCount), "We should have received 2000 messages from the server."); } } }
public void ConsumerShouldNotLoseMessageWhenBlocked() { var testId = Guid.NewGuid().ToString(); using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri))) using (var producer = new Producer(router)) { var offsets = producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).Result; //create consumer with buffer size of 1 (should block upstream) using (var consumer = new OldConsumer(new ConsumerOptions(TestConfig.TopicName(), router) { ConsumerBufferSize = 1, MaxWaitTimeForMinimumBytes = TimeSpan.Zero }, offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offset)).ToArray())) { for (var i = 0; i < 20; i++) { producer.SendMessageAsync(new Message(i.ToString(), testId), TestConfig.TopicName(), CancellationToken.None).Wait(); } for (var i = 0; i < 20; i++) { var result = consumer.Consume().Take(1).First(); Assert.That(result.Key.ToUtf8String(), Is.EqualTo(testId)); Assert.That(result.Value.ToUtf8String(), Is.EqualTo(i.ToString())); } } } }
public async Task ProducerAckLevel() { using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) using (var producer = new Producer(router)) { var responseAckLevel0 = await producer.SendMessageAsync(new Message("Ack Level 0"), TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 0), CancellationToken.None); Assert.AreEqual(responseAckLevel0.Offset, -1); var responseAckLevel1 = await producer.SendMessageAsync(new Message("Ack Level 1"), TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 1), CancellationToken.None); Assert.That(responseAckLevel1.Offset, Is.GreaterThan(-1)); } }
public async Task ConsumerShouldBeAbleToGetCurrentOffsetInformation() { var totalMessages = 20; var expected = totalMessages.Repeat(i => i.ToString()).ToList(); var testId = Guid.NewGuid().ToString(); using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) { using (var producer = new Producer(router)) { var offset = await producer.BrokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), 0, CancellationToken.None); for (var i = 0; i < totalMessages; i++) { await producer.SendMessageAsync(new Message(i.ToString(), testId), TestConfig.TopicName(), 0, CancellationToken.None); } using (var consumer = new Consumer(router, new ConsumerConfiguration(maxServerWait: TimeSpan.Zero))) { var results = await consumer.FetchMessagesAsync(offset, totalMessages, CancellationToken.None); TestConfig.InfoLog.Info(() => LogEvent.Create($"Message order: {string.Join(", ", results.Select(x => x.Value.ToUtf8String()).ToList())}")); Assert.That(results.Count, Is.EqualTo(totalMessages)); Assert.That(results.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected), "Expected the message list in the correct order."); var newOffset = await producer.BrokerRouter.GetTopicOffsetAsync(offset.TopicName, offset.PartitionId, CancellationToken.None); Assert.That(newOffset.Offset - offset.Offset, Is.EqualTo(totalMessages)); } } } }