public async Task TestSimpleConsumerWorksOk()
{
var keySerializer = new NullSerializer <object>();
var valueSerializer = new StringSerializer();
var messagePartitioner = new LoadBalancedPartitioner <object>();
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris()))
{
var topic = temporaryTopic.Name;
var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner);
var consumer = KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer,
new TopicSelector {
Partition = 0, Topic = topic
});
await producer.SendAsync(KeyedMessage.Create(topic, "Message"), CancellationToken.None);
var responses = await consumer.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(1));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(0));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("Message"));
}
}
public async Task ProduceToMultiplePartitions(int numberOfPartitions, int numberOfKeys, int numberOfMessages)
{
var keySerializer = new Int32Serializer();
var valueSerializer = new StringSerializer();
var messagePartitioner = new Int32Partitioner();
using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions: 2))
using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris()))
{
var topic = temporaryTopic.Name;
{
var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner);
var messages =
Enumerable
.Range(0, numberOfMessages)
.Select(i => KeyedMessage.Create(topic, i % numberOfKeys, i % numberOfPartitions, "Message " + i));
await producer.SendAsync(messages, CancellationToken.None);
}
{
var selectors =
Enumerable
.Range(0, numberOfPartitions)
.Select(partition => new TopicSelector {
Partition = partition, Topic = topic
})
.ToArray();
var consumer = KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer, selectors);
var responses = await consumer.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Has.Count.EqualTo(numberOfMessages));
var received = new bool[numberOfMessages];
var offsets = new long[numberOfPartitions];
foreach (var response in responses)
{
var split = response.Value.Split(' ');
Assert.That(split, Has.Length.EqualTo(2));
Assert.That(split[0], Is.EqualTo("Message"));
int messageNumber;
var parsed = Int32.TryParse(split[1], out messageNumber);
Assert.That(parsed, Is.True);
Assert.That(messageNumber, Is.InRange(0, numberOfMessages - 1));
var key = messageNumber % numberOfKeys;
Assert.That(response.Key, Is.EqualTo(key));
var partition = messageNumber % numberOfPartitions;
Assert.That(response.Partition, Is.EqualTo(partition));
Assert.That(received[messageNumber], Is.False);
received[messageNumber] = true;
Assert.That(response.Offset, Is.EqualTo(offsets[response.Partition]));
offsets[response.Partition] += 1;
Assert.That(response.Topic, Is.EqualTo(topic));
}
}
}
}
public async Task VerifyABrokerStoppingAndRestartingCanBeHandledByTheConsumer()
{
using (var cluster = new KafkaTestCluster("server.home", 1))
{
var topic = "test";
cluster.CreateTopic(topic);
using (var brokers = new KafkaBrokers(cluster.CreateBrokerUris()))
{
var producer = KafkaProducer.Create(brokers, new StringSerializer());
await producer.SendAsync(KeyedMessage.Create(topic, "Test"), CancellationToken.None);
await Task.Delay(1000);
cluster.StopKafkaBroker(0);
cluster.RestartKafkaBroker(0);
var consumer = KafkaConsumer.Create(topic, brokers, new StringSerializer(),
new TopicSelector {
Topic = topic, Partition = 0, Offset = 0
});
var result = await consumer.ReceiveAsync(CancellationToken.None);
Assert.That(result, Has.Count.EqualTo(1));
var first = result[0];
Assert.That(first.Value, Is.EqualTo("Test"));
Assert.That(first.Offset, Is.EqualTo(0));
}
cluster.DeleteTopic(topic);
}
}
public DefaultKafkaConsumerProvider(KafkaConsumerOptions kafkaConsumerOptions, Action <RecieveResult> onMessageRecieved)
{
this.kafkaConsumerOptions = kafkaConsumerOptions;
this.onMessageRecieved = onMessageRecieved;
Consumer = KafkaConsumer.Create(kafkaConsumerOptions.GroupId, kafkaConsumerOptions);
Consumer.EnableAutoCommit = kafkaConsumerOptions.EnableAutoCommit;
}
public async Task ProduceToTwoPartitions()
{
var keySerializer = new Int32Serializer();
var valueSerializer = new StringSerializer();
var messagePartitioner = new Int32Partitioner();
using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions: 2))
using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris()))
{
var topic = temporaryTopic.Name;
var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner);
var consumers = new[] {
KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer,
new TopicSelector {
Partition = 0, Topic = topic
}),
KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer,
new TopicSelector {
Partition = 1, Topic = topic
})
};
await producer.SendAsync(new[] {
KeyedMessage.Create(topic, 0, "Message to partition 0"),
KeyedMessage.Create(topic, 1, "Message to partition 1")
}, CancellationToken.None);
for (var i = 0; i < consumers.Length; i++)
{
var responses = await consumers[i].ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(1));
var first = responses.First();
Assert.That(first.Offset, Is.EqualTo(0));
Assert.That(first.Partition, Is.EqualTo(i));
Assert.That(first.Key, Is.EqualTo(i));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("Message to partition " + i));
}
}
}
public async Task TestProducing3MessagesAllowsTheConsumerToChooseTheCorrectMessage()
{
var valueSerializer = new StringSerializer();
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris()))
{
var topic = temporaryTopic.Name;
{
var producer = KafkaProducer.Create(brokers, valueSerializer);
await producer.SendAsync(new[] {
KeyedMessage.Create(topic, "1"),
KeyedMessage.Create(topic, "2"),
KeyedMessage.Create(topic, "3"),
}, CancellationToken.None);
}
{
var earliest = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector {
Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Earliest
});
var responses = await earliest.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(3));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(0));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("1"));
}
{
var latest = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector {
Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Last
});
var responses = await latest.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(1));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(2));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("3"));
}
{
var latest = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector {
Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Next
});
var responses = await latest.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(0));
}
{
var specified = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector {
Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Specified, Offset = 1
});
var responses = await specified.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(2));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(1));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("2"));
}
}
}
public async Task TestProducing3MessagesAllowsTheConsumerToCommitAndRestart()
{
var valueSerializer = new StringSerializer();
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris()))
{
var topic = temporaryTopic.Name;
{
var producer = KafkaProducer.Create(brokers, valueSerializer);
await producer.SendAsync(new[] {
KeyedMessage.Create(topic, "1"),
KeyedMessage.Create(topic, "2"),
KeyedMessage.Create(topic, "3"),
}, CancellationToken.None);
}
{
var noPreviousCommits = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector {
Partition = 0, Topic = topic,
DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted,
FailureOffsetSelection = OffsetSelectionStrategy.Earliest
});
var responses = await noPreviousCommits.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(3));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(0));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("1"));
await noPreviousCommits.CommitAsync(new[] {
new TopicPartitionOffset {
Topic = topic, Partition = 0, Offset = 0
}
}, CancellationToken.None);;
}
{
var previousCommit = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector
{
Partition = 0,
Topic = topic,
DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted,
FailureOffsetSelection = OffsetSelectionStrategy.Earliest
});
var responses = await previousCommit.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(2));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(1));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("2"));
}
{
var previousCommitAgain = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector
{
Partition = 0,
Topic = topic,
DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted,
FailureOffsetSelection = OffsetSelectionStrategy.Earliest
});
var responses = await previousCommitAgain.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(2));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(1));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("2"));
await previousCommitAgain.CommitAsync(new[] {
new TopicPartitionOffset {
Topic = topic, Partition = 0, Offset = 1
}
}, CancellationToken.None);;
}
{
var secondCommit = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector
{
Partition = 0,
Topic = topic,
DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted,
FailureOffsetSelection = OffsetSelectionStrategy.Earliest
});
var responses = await secondCommit.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(1));
var first = responses.First();
Assert.That(first.Key, Is.Null);
Assert.That(first.Offset, Is.EqualTo(2));
Assert.That(first.Partition, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Value, Is.EqualTo("3"));
await secondCommit.CommitAsync(new[] {
new TopicPartitionOffset {
Topic = topic, Partition = 0, Offset = 2
}
}, CancellationToken.None);;
}
{
var thirdCommit = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer,
new TopicSelector
{
Partition = 0,
Topic = topic,
DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted,
FailureOffsetSelection = OffsetSelectionStrategy.Earliest
});
var responses = await thirdCommit.ReceiveAsync(CancellationToken.None);
Assert.That(responses, Is.Not.Null);
Assert.That(responses, Has.Count.EqualTo(0));
}
}
}