public async Task TestSimpleConsumerWorksOk() { var keySerializer = new NullSerializer <object>(); var valueSerializer = new StringSerializer(); var messagePartitioner = new LoadBalancedPartitioner <object>(); using (var temporaryTopic = testCluster.CreateTemporaryTopic()) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var topic = temporaryTopic.Name; var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner); var consumer = KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer, new TopicSelector { Partition = 0, Topic = topic }); await producer.SendAsync(KeyedMessage.Create(topic, "Message"), CancellationToken.None); var responses = await consumer.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(1)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(0)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("Message")); } }
public async Task ProduceToMultiplePartitions(int numberOfPartitions, int numberOfKeys, int numberOfMessages) { var keySerializer = new Int32Serializer(); var valueSerializer = new StringSerializer(); var messagePartitioner = new Int32Partitioner(); using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions: 2)) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var topic = temporaryTopic.Name; { var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner); var messages = Enumerable .Range(0, numberOfMessages) .Select(i => KeyedMessage.Create(topic, i % numberOfKeys, i % numberOfPartitions, "Message " + i)); await producer.SendAsync(messages, CancellationToken.None); } { var selectors = Enumerable .Range(0, numberOfPartitions) .Select(partition => new TopicSelector { Partition = partition, Topic = topic }) .ToArray(); var consumer = KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer, selectors); var responses = await consumer.ReceiveAsync(CancellationToken.None); Assert.That(responses, Has.Count.EqualTo(numberOfMessages)); var received = new bool[numberOfMessages]; var offsets = new long[numberOfPartitions]; foreach (var response in responses) { var split = response.Value.Split(' '); Assert.That(split, Has.Length.EqualTo(2)); Assert.That(split[0], Is.EqualTo("Message")); int messageNumber; var parsed = Int32.TryParse(split[1], out messageNumber); Assert.That(parsed, Is.True); Assert.That(messageNumber, Is.InRange(0, numberOfMessages - 1)); var key = messageNumber % numberOfKeys; Assert.That(response.Key, Is.EqualTo(key)); var partition = messageNumber % numberOfPartitions; Assert.That(response.Partition, Is.EqualTo(partition)); Assert.That(received[messageNumber], Is.False); received[messageNumber] = true; Assert.That(response.Offset, Is.EqualTo(offsets[response.Partition])); offsets[response.Partition] += 1; Assert.That(response.Topic, Is.EqualTo(topic)); } } } }
public async Task VerifyABrokerStoppingAndRestartingCanBeHandledByTheConsumer() { using (var cluster = new KafkaTestCluster("server.home", 1)) { var topic = "test"; cluster.CreateTopic(topic); using (var brokers = new KafkaBrokers(cluster.CreateBrokerUris())) { var producer = KafkaProducer.Create(brokers, new StringSerializer()); await producer.SendAsync(KeyedMessage.Create(topic, "Test"), CancellationToken.None); await Task.Delay(1000); cluster.StopKafkaBroker(0); cluster.RestartKafkaBroker(0); var consumer = KafkaConsumer.Create(topic, brokers, new StringSerializer(), new TopicSelector { Topic = topic, Partition = 0, Offset = 0 }); var result = await consumer.ReceiveAsync(CancellationToken.None); Assert.That(result, Has.Count.EqualTo(1)); var first = result[0]; Assert.That(first.Value, Is.EqualTo("Test")); Assert.That(first.Offset, Is.EqualTo(0)); } cluster.DeleteTopic(topic); } }
public async Task TestSimpleProducerWorksOk() { var valueSerializer = new StringSerializer(); using (var temporaryTopic = testCluster.CreateTemporaryTopic()) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var producer = KafkaProducer.Create(brokers, valueSerializer); await producer.SendAsync(KeyedMessage.Create(temporaryTopic.Name, "Message"), CancellationToken.None); } }
public async Task ProduceToTwoPartitions() { var keySerializer = new Int32Serializer(); var valueSerializer = new StringSerializer(); var messagePartitioner = new Int32Partitioner(); using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions: 2)) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var topic = temporaryTopic.Name; var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner); var consumers = new[] { KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer, new TopicSelector { Partition = 0, Topic = topic }), KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer, new TopicSelector { Partition = 1, Topic = topic }) }; await producer.SendAsync(new[] { KeyedMessage.Create(topic, 0, "Message to partition 0"), KeyedMessage.Create(topic, 1, "Message to partition 1") }, CancellationToken.None); for (var i = 0; i < consumers.Length; i++) { var responses = await consumers[i].ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(1)); var first = responses.First(); Assert.That(first.Offset, Is.EqualTo(0)); Assert.That(first.Partition, Is.EqualTo(i)); Assert.That(first.Key, Is.EqualTo(i)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("Message to partition " + i)); } } }
public async Task TestProducing3MessagesAllowsTheConsumerToChooseTheCorrectMessage() { var valueSerializer = new StringSerializer(); using (var temporaryTopic = testCluster.CreateTemporaryTopic()) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var topic = temporaryTopic.Name; { var producer = KafkaProducer.Create(brokers, valueSerializer); await producer.SendAsync(new[] { KeyedMessage.Create(topic, "1"), KeyedMessage.Create(topic, "2"), KeyedMessage.Create(topic, "3"), }, CancellationToken.None); } { var earliest = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Earliest }); var responses = await earliest.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(3)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(0)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("1")); } { var latest = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Last }); var responses = await latest.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(1)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(2)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("3")); } { var latest = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Next }); var responses = await latest.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(0)); } { var specified = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.Specified, Offset = 1 }); var responses = await specified.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(2)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(1)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("2")); } } }
public async Task TestProducing3MessagesAllowsTheConsumerToCommitAndRestart() { var valueSerializer = new StringSerializer(); using (var temporaryTopic = testCluster.CreateTemporaryTopic()) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var topic = temporaryTopic.Name; { var producer = KafkaProducer.Create(brokers, valueSerializer); await producer.SendAsync(new[] { KeyedMessage.Create(topic, "1"), KeyedMessage.Create(topic, "2"), KeyedMessage.Create(topic, "3"), }, CancellationToken.None); } { var noPreviousCommits = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted, FailureOffsetSelection = OffsetSelectionStrategy.Earliest }); var responses = await noPreviousCommits.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(3)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(0)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("1")); await noPreviousCommits.CommitAsync(new[] { new TopicPartitionOffset { Topic = topic, Partition = 0, Offset = 0 } }, CancellationToken.None);; } { var previousCommit = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted, FailureOffsetSelection = OffsetSelectionStrategy.Earliest }); var responses = await previousCommit.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(2)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(1)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("2")); } { var previousCommitAgain = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted, FailureOffsetSelection = OffsetSelectionStrategy.Earliest }); var responses = await previousCommitAgain.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(2)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(1)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("2")); await previousCommitAgain.CommitAsync(new[] { new TopicPartitionOffset { Topic = topic, Partition = 0, Offset = 1 } }, CancellationToken.None);; } { var secondCommit = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted, FailureOffsetSelection = OffsetSelectionStrategy.Earliest }); var responses = await secondCommit.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(1)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(2)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("3")); await secondCommit.CommitAsync(new[] { new TopicPartitionOffset { Topic = topic, Partition = 0, Offset = 2 } }, CancellationToken.None);; } { var thirdCommit = KafkaConsumer.Create(defaultConsumerGroup, brokers, valueSerializer, new TopicSelector { Partition = 0, Topic = topic, DefaultOffsetSelection = OffsetSelectionStrategy.NextUncommitted, FailureOffsetSelection = OffsetSelectionStrategy.Earliest }); var responses = await thirdCommit.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(0)); } } }