public async Task CanFetch()
{
int partitionId = 0;
var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri));
var producer = new Producer(router);
string messageValue = Guid.NewGuid().ToString();
var response = await producer.SendMessageAsync(new Message(messageValue), TestConfig.TopicName(), partitionId, CancellationToken.None);
var offset = response.Offset;
var fetch = new FetchRequest.Topic(TestConfig.TopicName(), partitionId, offset, 32000);
var fetchRequest = new FetchRequest(fetch, minBytes: 10);
var r = await router.SendAsync(fetchRequest, TestConfig.TopicName(), partitionId, CancellationToken.None);
Assert.IsTrue(r.Topics.First().Messages.First().Value.ToUtf8String() == messageValue);
}
public async Task SendAsyncShouldHandleHighVolumeOfMessages(int amount, int maxAsync)
{
using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri)))
using (var producer = new Producer(router, new ProducerConfiguration(maxAsync, amount / 2)))
{
var tasks = new Task <ProduceResponse.Topic> [amount];
for (var i = 0; i < amount; i++)
{
tasks[i] = producer.SendMessageAsync(new Message(Guid.NewGuid().ToString()), TestConfig.TopicName(), CancellationToken.None);
}
var results = await Task.WhenAll(tasks.ToArray());
//Because of how responses are batched up and sent to servers, we will usually get multiple responses per requested message batch
//So this assertion will never pass
//Assert.That(results.Count, Is.EqualTo(amount));
Assert.That(results.Any(x => x.ErrorCode != ErrorResponseCode.None), Is.False,
"Should not have received any results as failures.");
}
}
public async Task FetchMessagesInvalidOffsetTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
var consumer = new Consumer(brokerRouter, _consumerConfig);
var offset = -1;
// Now let's consume
Assert.ThrowsAsync <ArgumentOutOfRangeException>(async() => await consumer.FetchMessagesAsync(TestConfig.TopicName(), _partitionId, offset, 5, CancellationToken.None));
}
public async Task ConsumeByOffsetShouldGetSameMessageProducedAtSameOffset()
{
long offsetResponse;
var messge = Guid.NewGuid();
using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) {
using (var producer = new Producer(router)) {
var responseAckLevel1 = await producer.SendMessageAsync(new Message(messge.ToString()), TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 1), CancellationToken.None);
offsetResponse = responseAckLevel1.Offset;
}
}
using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) {
using (var consumer = new Consumer(router, new ConsumerConfiguration(maxServerWait: TimeSpan.Zero))) {
var result = await consumer.FetchMessagesAsync(TestConfig.TopicName(), 0, offsetResponse, 1, CancellationToken.None);
Assert.AreEqual(messge.ToString(), result[0].Value.ToUtf8String());
}
}
}
public async Task ConsumerShouldConsumeInSameOrderAsProduced([Values(20)] int sends, [Values(1, 10)] int messagesPerSend)
{
var totalMessages = sends * messagesPerSend;
var expected = totalMessages.Repeat(i => i.ToString()).ToList();
var testId = Guid.NewGuid().ToString();
using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) {
using (var producer = new Producer(router)) {
var offset = await producer.BrokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), 0, CancellationToken.None);
for (var i = 0; i < sends; i++)
{
if (messagesPerSend == 1)
{
await producer.SendMessageAsync(new Message(i.ToString(), testId), TestConfig.TopicName(), 0, CancellationToken.None);
}
else
{
var current = i * messagesPerSend;
var messages = messagesPerSend.Repeat(_ => new Message((current + _).ToString(), testId)).ToList();
await producer.SendMessagesAsync(messages, TestConfig.TopicName(), 0, CancellationToken.None);
}
}
using (var consumer = new Consumer(router, new ConsumerConfiguration(maxServerWait: TimeSpan.Zero))) {
var results = await consumer.FetchMessagesAsync(offset, totalMessages, CancellationToken.None);
//ensure the produced messages arrived
TestConfig.InfoLog.Info(() => LogEvent.Create($"Message order: {string.Join(", ", results.Select(x => x.Value.ToUtf8String()).ToList())}"));
Assert.That(results.Count, Is.EqualTo(totalMessages));
Assert.That(results.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected), "Expected the message list in the correct order.");
Assert.That(results.Any(x => x.Key.ToUtf8String() != testId), Is.False);
}
}
}
}
public async Task ProducerLastResposeOffsetAckLevel1ShouldBeEqualsToLastOffset()
{
using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog))
using (var producer = new Producer(router))
{
var responseAckLevel1 = await producer.SendMessagesAsync(new [] { new Message("Ack Level 1"), new Message("Ack Level 1") }, TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 1), CancellationToken.None);
var offsetResponse = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None);
var maxOffset = offsetResponse.First(x => x.PartitionId == 0);
Assert.AreEqual(responseAckLevel1.Last().Offset, maxOffset.Offset - 1);
}
}
public async Task ConsumerShouldMoveToNextAvailableOffsetWhenQueryingForNextMessage()
{
const int expectedCount = 1000;
var options = new KafkaOptions(TestConfig.IntegrationUri);
using (var producerRouter = new BrokerRouter(options))
using (var producer = new Producer(producerRouter))
{
//get current offset and reset consumer to top of log
var offsets = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).ConfigureAwait(false);
using (var consumerRouter = new BrokerRouter(options))
using (var consumer = new OldConsumer(new ConsumerOptions(TestConfig.TopicName(), consumerRouter)
{
MaxWaitTimeForMinimumBytes = TimeSpan.Zero
},
offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offset)).ToArray()))
{
Console.WriteLine("Sending {0} test messages", expectedCount);
var response = await producer.SendMessagesAsync(Enumerable.Range(0, expectedCount).Select(x => new Message(x.ToString())), TestConfig.TopicName(), CancellationToken.None);
Assert.That(response.Any(x => x.ErrorCode != (int)ErrorResponseCode.None), Is.False, "Error occured sending test messages to server.");
var stream = consumer.Consume();
Console.WriteLine("Reading message back out from consumer.");
var data = stream.Take(expectedCount).ToList();
var consumerOffset = consumer.GetOffsetPosition().OrderBy(x => x.PartitionId).ToList();
var serverOffset = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).ConfigureAwait(false);
var positionOffset = serverOffset.Select(x => new OffsetPosition(x.PartitionId, x.Offset))
.OrderBy(x => x.PartitionId)
.ToList();
Assert.That(consumerOffset, Is.EqualTo(positionOffset), "The consumerOffset position should match the server offset position.");
Assert.That(data.Count, Is.EqualTo(expectedCount), "We should have received 2000 messages from the server.");
}
}
}
public void ConsumerShouldNotLoseMessageWhenBlocked()
{
var testId = Guid.NewGuid().ToString();
using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var offsets = producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).Result;
//create consumer with buffer size of 1 (should block upstream)
using (var consumer = new OldConsumer(new ConsumerOptions(TestConfig.TopicName(), router)
{
ConsumerBufferSize = 1, MaxWaitTimeForMinimumBytes = TimeSpan.Zero
},
offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offset)).ToArray()))
{
for (var i = 0; i < 20; i++)
{
producer.SendMessageAsync(new Message(i.ToString(), testId), TestConfig.TopicName(), CancellationToken.None).Wait();
}
for (var i = 0; i < 20; i++)
{
var result = consumer.Consume().Take(1).First();
Assert.That(result.Key.ToUtf8String(), Is.EqualTo(testId));
Assert.That(result.Value.ToUtf8String(), Is.EqualTo(i.ToString()));
}
}
}
}
public async Task ProducerAckLevel()
{
using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog))
using (var producer = new Producer(router))
{
var responseAckLevel0 = await producer.SendMessageAsync(new Message("Ack Level 0"), TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 0), CancellationToken.None);
Assert.AreEqual(responseAckLevel0.Offset, -1);
var responseAckLevel1 = await producer.SendMessageAsync(new Message("Ack Level 1"), TestConfig.TopicName(), 0, new SendMessageConfiguration(acks : 1), CancellationToken.None);
Assert.That(responseAckLevel1.Offset, Is.GreaterThan(-1));
}
}
public async Task ConsumerShouldBeAbleToGetCurrentOffsetInformation()
{
var totalMessages = 20;
var expected = totalMessages.Repeat(i => i.ToString()).ToList();
var testId = Guid.NewGuid().ToString();
using (var router = new BrokerRouter(TestConfig.IntegrationUri, log: TestConfig.InfoLog)) {
using (var producer = new Producer(router)) {
var offset = await producer.BrokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), 0, CancellationToken.None);
for (var i = 0; i < totalMessages; i++)
{
await producer.SendMessageAsync(new Message(i.ToString(), testId), TestConfig.TopicName(), 0, CancellationToken.None);
}
using (var consumer = new Consumer(router, new ConsumerConfiguration(maxServerWait: TimeSpan.Zero))) {
var results = await consumer.FetchMessagesAsync(offset, totalMessages, CancellationToken.None);
TestConfig.InfoLog.Info(() => LogEvent.Create($"Message order: {string.Join(", ", results.Select(x => x.Value.ToUtf8String()).ToList())}"));
Assert.That(results.Count, Is.EqualTo(totalMessages));
Assert.That(results.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected), "Expected the message list in the correct order.");
var newOffset = await producer.BrokerRouter.GetTopicOffsetAsync(offset.TopicName, offset.PartitionId, CancellationToken.None);
Assert.That(newOffset.Offset - offset.Offset, Is.EqualTo(totalMessages));
}
}
}
}