/// <summary>
/// Get offsets for a single partitions of a given topic.
/// </summary>
/// <param name="brokerRouter">The router which provides the route and metadata.</param>
/// <param name="topicName">Name of the topic to get offset information from.</param>
/// <param name="partitionId">The partition to get offsets for.</param>
/// <param name="consumerGroup">The id of the consumer group</param>
/// <param name="cancellationToken"></param>
public static async Task <OffsetFetchResponse.Topic> GetTopicOffsetAsync(this IBrokerRouter brokerRouter, string topicName, int partitionId, string consumerGroup, CancellationToken cancellationToken)
{
var request = new OffsetFetchRequest(consumerGroup, new TopicPartition(topicName, partitionId));
var response = await brokerRouter.SendAsync(request, topicName, partitionId, consumerGroup, cancellationToken).ConfigureAwait(false);
return(response.Topics.SingleOrDefault(t => t.TopicName == topicName && t.PartitionId == partitionId));
}
public async Task TestMultipleOffsetFetchesWorkOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions: 2))
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
var request = new OffsetFetchRequest
{
Topics =
Enumerable
.Range(0, 2)
.Select(i => new OffsetFetch
{
Topic = topic,
PartitionId = i
})
.ToList()
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(2));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
Assert.That(response[1].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}
/// <summary>
/// Get group offsets for a single partition of a given topic.
/// </summary>
/// <param name="router">The router which provides the route and metadata.</param>
/// <param name="topicName">Name of the topic to get offset information from.</param>
/// <param name="partitionId">The partition to get offsets for.</param>
/// <param name="groupId">The id of the consumer group</param>
/// <param name="cancellationToken"></param>
public static async Task <OffsetFetchResponse.Topic> GetOffsetsAsync(this IRouter router, string groupId, string topicName, int partitionId, CancellationToken cancellationToken)
{
var request = new OffsetFetchRequest(groupId, new TopicPartition(topicName, partitionId));
var response = await router.SendAsync(request, topicName, partitionId, cancellationToken).ConfigureAwait(false);
return(response.responses.SingleOrDefault(t => t.topic == topicName && t.partition_id == partitionId));
}
public async Task OffsetCommitShouldStoreOffsetValue()
{
const int partitionId = 0;
const long offset = 99;
using (var router = await TestConfig.IntegrationOptions.CreateRouterAsync()) {
await router.TemporaryTopicAsync(async topicName => {
await router.GetTopicMetadataAsync(topicName, CancellationToken.None);
var conn = router.GetTopicConnection(topicName, partitionId);
// ensure the group exists
var groupId = TestConfig.GroupId();
var group = new GroupCoordinatorRequest(groupId);
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.error_code, Is.EqualTo(ErrorCode.NONE));
var commit = new OffsetCommitRequest(group.group_id, new [] { new OffsetCommitRequest.Topic(topicName, partitionId, offset, null) });
var commitResponse = await conn.Connection.SendAsync(commit, CancellationToken.None);
var commitTopic = commitResponse.responses.SingleOrDefault();
Assert.That(commitTopic, Is.Not.Null);
Assert.That(commitTopic.error_code, Is.EqualTo(ErrorCode.NONE));
var fetch = new OffsetFetchRequest(groupId, new TopicPartition(topicName, partitionId));
var fetchResponse = await conn.Connection.SendAsync(fetch, CancellationToken.None);
var fetchTopic = fetchResponse.responses.SingleOrDefault();
Assert.That(fetchTopic, Is.Not.Null);
Assert.That(fetchTopic.error_code, Is.EqualTo(ErrorCode.NONE));
Assert.That(fetchTopic.offset, Is.EqualTo(offset));
});
}
}
private static IEnumerable <OffsetFetchResponse> OffsetFetchRequest(IEnumerable <OffsetFetch> offsetFetches)
{
var request = new OffsetFetchRequest
{
ConsumerGroup = ConsumerGroup,
Topics = offsetFetches.ToList()
};
return(_brokerRoute.Connection.SendAsync(request).Result);
}
public void OffsetFetchRequest()
{
var request = new OffsetFetchRequest();
request.ConsumerGroup = Guid.NewGuid().ToString();
request.TopicPartitions = new[] {
new OffsetFetchRequestTopicPartition {
TopicName = Guid.NewGuid().ToString(),
Partitions = new[] { _random.Next() }
}
};
Stream binary1 = new MemoryStream();
request.Serialize(binary1);
binary1.Seek(0L, SeekOrigin.Begin);
var request2 = new OffsetFetchRequest();
request2.Deserialize(binary1);
var compareLogic = new CompareLogic();
var result = compareLogic.Compare(request, request2);
Assert.True(result.AreEqual);
Stream binary2 = new MemoryStream();
request.Serialize(binary2);
Assert.Equal(binary1.Length, binary2.Length);
using (var stream1 = new MemoryStream())
using (var stream2 = new MemoryStream()) {
binary1.Seek(0L, SeekOrigin.Begin);
binary1.CopyTo(stream1);
binary2.Seek(0L, SeekOrigin.Begin);
binary2.CopyTo(stream2);
Assert.Equal(stream1.Length, stream2.Length);
stream1.Seek(0L, SeekOrigin.Begin);
var bytes1 = stream1.ToArray();
stream2.Seek(0L, SeekOrigin.Begin);
var bytes2 = stream2.ToArray();
Assert.Equal(bytes1.Length, bytes2.Length);
for (int i = 0; i < bytes1.Length; i++)
{
Assert.Equal(bytes1[i], bytes2[i]);
}
}
}
/// <summary>
/// Getting the offset of a specific consumer group
/// </summary>
/// <param name="consumerGroup">The name of the consumer group</param>
/// <returns>The current offset of the consumerGroup</returns>
public async Task <long> FetchOffset(string consumerGroup)
{
if (string.IsNullOrEmpty(consumerGroup))
{
throw new ArgumentNullException("consumerGroup");
}
OffsetFetchRequest offsetFetchRequest = CreateOffsetFetchRequest(consumerGroup);
var response = await _gateway.SendProtocolRequest(offsetFetchRequest, _topic, _partitionId).ConfigureAwait(false);
return(response.Offset);
}
public void OffsetFetchRequest(
[Values("group1", "group2")] string groupId,
[Values("test", "a really long name, with spaces and punctuation!")] string topic,
[Values(1, 10)] int topicsPerRequest,
[Values(5)] int maxPartitions)
{
var topics = new List <TopicPartition>();
for (var t = 0; t < topicsPerRequest; t++)
{
topics.Add(new TopicPartition(topic + t, t % maxPartitions));
}
var request = new OffsetFetchRequest(groupId, topics);
request.AssertCanEncodeDecodeRequest(0);
}
private OffsetFetchRequest CreateOffsetFetchRequest(string consumerGroup, int partitionId)
{
var request = new OffsetFetchRequest
{
ConsumerGroup = consumerGroup,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = partitionId,
Topic = IntegrationConfig.IntegrationTopic
}
}
};
return(request);
}
private OffsetFetchRequest CreateOffsetFetchRequest(string consumerGroup)
{
OffsetFetch topicFetch = new OffsetFetch()
{
PartitionId = _partitionId, Topic = _topic
};
OffsetFetchRequest request = new OffsetFetchRequest()
{
ConsumerGroup = consumerGroup,
Topics = new List <OffsetFetch>()
{
topicFetch
},
ClientId = _clientId
};
return(request);
}
private OffsetFetchResponse GetOffset(string topic)
{
var request = new OffsetFetchRequest
{
ConsumerGroup = ConsumerGroupName,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
return(_router.SelectBrokerRoute(topic).Connection.SendAsync(request).Result.FirstOrDefault());
}
private static OffsetFetchResponse OffsetFetchRequest()
{
var request = new OffsetFetchRequest
{
ConsumerGroup = ConsumerGroup,
Topics = new List <OffsetFetch>
{
new OffsetFetch {
PartitionId = PartitionId, Topic = Topic
}
}
};
var response = _brokerRoute.Connection.SendAsync(request).Result.FirstOrDefault();
Console.WriteLine(JsonConvert.SerializeObject(response));
return(response);
}
public Int64 OffsetFetch(String topic, Int32 partition, String groupId)
{
EnsureLegalTopicSpelling(topic);
var broker = _topicBrokerDispatcher.SelectBroker(topic, partition);
var request = new OffsetFetchRequest(topic, new[] { partition }, groupId);
var response = (OffsetFetchResponse)SubmitRequest(broker, request);
var errors = response.TopicPartitions
.SelectMany(r => r.Details)
.Where(x => x.ErrorCode != ErrorCode.NoError)
.ToList();
if (errors.Count > 0 && errors.All(e => e.ErrorCode == ErrorCode.UnknownTopicOrPartition))
{
return(-1L);
}
if (errors.Count > 1)
{
throw new ProtocolException(errors.First().ErrorCode);
}
return(response.TopicPartitions[0].Details[0].Offset);
}
public async Task TestOffsetFetchWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var request = new OffsetFetchRequest
{
Topics = new List <OffsetFetch>
{
new OffsetFetch {
Topic = temporaryTopic.Name,
PartitionId = 0
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}
public async Task OffsetFetchRequestOfNonExistingGroupShouldReturnNoError()
{
//From documentation: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+ProtocolTests#AGuideToTheKafkaProtocol-OffsetFetchRequest
//Note that if there is no offset associated with a topic-partition under that consumer group the broker does not set an error code
//(since it is not really an error), but returns empty metadata and sets the offset field to -1.
const int partitionId = 0;
var router = new BrokerRouter(_options);
var request = new OffsetFetchRequest(Guid.NewGuid().ToString(), new TopicPartition(TestConfig.TopicName(), partitionId));
await router.GetTopicMetadataAsync(TestConfig.TopicName(), CancellationToken.None);
var conn = router.GetBrokerRoute(TestConfig.TopicName(), partitionId);
var response = await conn.Connection.SendAsync(request, CancellationToken.None);
var topic = response.Topics.FirstOrDefault();
Assert.That(topic, Is.Not.Null);
Assert.That(topic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
Assert.That(topic.Offset, Is.EqualTo(-1));
router.Dispose();
}
public async Task OffsetCommitShouldStoreOffsetValue()
{
const int partitionId = 0;
const long offset = 99;
var router = new BrokerRouter(_options);
await router.GetTopicMetadataAsync(TestConfig.TopicName(), CancellationToken.None);
var conn = router.GetBrokerRoute(TestConfig.TopicName(), partitionId);
// ensure the group exists
var group = new GroupCoordinatorRequest(TestConfig.ConsumerName());
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var commit = new OffsetCommitRequest(group.GroupId, new [] { new OffsetCommitRequest.Topic(TestConfig.TopicName(), partitionId, offset, null) });
var commitResponse = await conn.Connection.SendAsync(commit, CancellationToken.None);
var commitTopic = commitResponse.Topics.SingleOrDefault();
Assert.That(commitTopic, Is.Not.Null);
Assert.That(commitTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var fetch = new OffsetFetchRequest(TestConfig.ConsumerName(), new TopicPartition(TestConfig.TopicName(), partitionId));
var fetchResponse = await conn.Connection.SendAsync(fetch, CancellationToken.None);
var fetchTopic = fetchResponse.Topics.SingleOrDefault();
Assert.That(fetchTopic, Is.Not.Null);
Assert.That(fetchTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
Assert.That(fetchTopic.Offset, Is.EqualTo(offset));
router.Dispose();
}
private OffsetFetchRequest CreateOffsetFetchRequest(string consumerGroup, int partitionId)
{
var request = new OffsetFetchRequest
{
ConsumerGroup = consumerGroup,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = partitionId,
Topic = IntegrationConfig.IntegrationTopic
}
}
};
return request;
}
public async Task TestOffsetFetchWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var request = new OffsetFetchRequest
{
Topics = new List<OffsetFetch>
{
new OffsetFetch {
Topic = temporaryTopic.Name,
PartitionId = 0
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}
private OffsetFetchResponse GetOffset(string topic)
{
var request = new OffsetFetchRequest
{
ConsumerGroup = ConsumerGroupName,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
return _router.SelectBrokerRoute(topic).Connection.SendAsync(request).Result.FirstOrDefault();
}
public async Task TestNewTopicProductionWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
{
var request = new MetadataRequest
{
Topics = new List <string>
{
topic
}
};
MetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Topics[0].ErrorCode == ErrorResponseCode.LeaderNotAvailable)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response, Is.Not.Null);
var first = response;
Assert.That(first.Topics, Has.Length.EqualTo(1));
var firstTopic = first.Topics.First();
Assert.That(firstTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(firstTopic.Name, Is.EqualTo(topic));
Assert.That(firstTopic.Partitions, Has.Length.EqualTo(1));
var firstPartition = firstTopic.Partitions.First();
Assert.That(firstPartition.PartitionId, Is.EqualTo(0));
}
{
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 10000,
Payload = new List <Payload>
{
new Payload
{
Topic = topic,
Partition = 0,
Codec = MessageCodec.CodecNone,
Messages = new List <Message>
{
new Message("Message 1"),
new Message("Message 2"),
new Message("Message 3"),
new Message("Message 4"),
}
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Is.Not.Null);
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 40,
Offset = 0,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(1));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(0));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 1"));
}
{
var request = new OffsetRequest
{
Offsets = new List <Offset>
{
new Offset
{
MaxOffsets = 2,
PartitionId = 0,
Time = -1,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offsets, Has.Length.EqualTo(2));
Assert.That(first.Offsets[0], Is.EqualTo(4));
Assert.That(first.Offsets[1], Is.EqualTo(0));
}
{
var request = new ConsumerMetadataRequest
{
ConsumerGroup = topic
};
ConsumerMetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Error == ErrorResponseCode.ConsumerCoordinatorNotAvailableCode)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response.Error, Is.EqualTo(ErrorResponseCode.NoError));
Console.WriteLine("Id = {0}, Host = {1}, Port = {2}", response.CoordinatorId, response.CoordinatorHost, response.CoordinatorPort);
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.Empty);
Assert.That(first.Offset, Is.EqualTo(-1));
}
{
var request = new OffsetCommitRequest
{
ConsumerGroup = topic,
ConsumerGroupGenerationId = 1,
ConsumerId = "0",
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
Metadata = "Metadata 1",
Offset = 0,
PartitionId = 0,
TimeStamp = -1,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.EqualTo("Metadata 1"));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 0 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(3));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(1));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 2"));
var lastMessage = first.Messages.Last();
Assert.That(lastMessage.Meta.Offset, Is.EqualTo(3));
Assert.That(lastMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(lastMessage.Attribute, Is.EqualTo(0));
Assert.That(lastMessage.Key, Is.Null);
Assert.That(lastMessage.MagicNumber, Is.EqualTo(0));
Assert.That(lastMessage.Value, Is.Not.Null);
var lastString = lastMessage.Value.ToUtf8String();
Assert.That(lastString, Is.EqualTo("Message 4"));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 3 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(0));
}
}
Console.WriteLine("Test completed");
}
public async Task TestMultipleOffsetFetchesWorkOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions:2))
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
var request = new OffsetFetchRequest
{
Topics =
Enumerable
.Range(0, 2)
.Select(i => new OffsetFetch
{
Topic = topic,
PartitionId = i
})
.ToList()
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(2));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
Assert.That(response[1].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}
public async Task TestNewTopicProductionWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
{
var request = new MetadataRequest
{
Topics = new List<string>
{
topic
}
};
MetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Topics[0].ErrorCode == ErrorResponseCode.LeaderNotAvailable)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response, Is.Not.Null);
var first = response;
Assert.That(first.Topics, Has.Length.EqualTo(1));
var firstTopic = first.Topics.First();
Assert.That(firstTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(firstTopic.Name, Is.EqualTo(topic));
Assert.That(firstTopic.Partitions, Has.Length.EqualTo(1));
var firstPartition = firstTopic.Partitions.First();
Assert.That(firstPartition.PartitionId, Is.EqualTo(0));
}
{
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 10000,
Payload = new List<Payload>
{
new Payload
{
Topic = topic,
Partition = 0,
Codec = MessageCodec.CodecNone,
Messages = new List<Message>
{
new Message("Message 1"),
new Message("Message 2"),
new Message("Message 3"),
new Message("Message 4"),
}
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Is.Not.Null);
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 40,
Offset = 0,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(1));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(0));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 1"));
}
{
var request = new OffsetRequest
{
Offsets = new List<Offset>
{
new Offset
{
MaxOffsets = 2,
PartitionId = 0,
Time = -1,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offsets, Has.Length.EqualTo(2));
Assert.That(first.Offsets[0], Is.EqualTo(4));
Assert.That(first.Offsets[1], Is.EqualTo(0));
}
{
var request = new ConsumerMetadataRequest
{
ConsumerGroup = topic
};
ConsumerMetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Error == ErrorResponseCode.ConsumerCoordinatorNotAvailableCode)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response.Error, Is.EqualTo(ErrorResponseCode.NoError));
Console.WriteLine("Id = {0}, Host = {1}, Port = {2}", response.CoordinatorId, response.CoordinatorHost, response.CoordinatorPort);
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.Empty);
Assert.That(first.Offset, Is.EqualTo(-1));
}
{
var request = new OffsetCommitRequest
{
ConsumerGroup = topic,
ConsumerGroupGenerationId = 1,
ConsumerId = "0",
OffsetCommits = new List<OffsetCommit>
{
new OffsetCommit
{
Metadata = "Metadata 1",
Offset = 0,
PartitionId = 0,
TimeStamp = -1,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.EqualTo("Metadata 1"));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 0 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(3));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(1));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 2"));
var lastMessage = first.Messages.Last();
Assert.That(lastMessage.Meta.Offset, Is.EqualTo(3));
Assert.That(lastMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(lastMessage.Attribute, Is.EqualTo(0));
Assert.That(lastMessage.Key, Is.Null);
Assert.That(lastMessage.MagicNumber, Is.EqualTo(0));
Assert.That(lastMessage.Value, Is.Not.Null);
var lastString = lastMessage.Value.ToUtf8String();
Assert.That(lastString, Is.EqualTo("Message 4"));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 3 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(0));
}
}
Console.WriteLine("Test completed");
}
private OffsetFetchRequest CreateOffsetFetchRequest(string consumerGroup)
{
var topicFetch = new OffsetFetch() { PartitionId = _partitionId, Topic = _topic };
var request = new OffsetFetchRequest()
{
ConsumerGroup = consumerGroup,
Topics = new List<OffsetFetch>() { topicFetch },
ClientId = _clientId
};
return request;
}
public ValueTask <OffsetFetchResponse> OffsetFetch(OffsetFetchRequest request) => SendRequest(ApiKey.OffsetFetch, 1, request, new OffsetFetchRequestWriter(), new OffsetFetchResponseReader());
