public async Task <long> CommitMarkedAsync(CancellationToken cancellationToken)
{
if (ReferenceEquals(this, Empty))
{
return(0L);
}
if (_disposeCount > 0)
{
throw new ObjectDisposedException($"The {_partition} batch is disposed.");
}
var offset = _offsetMarked;
var committed = _offsetCommitted;
if (offset <= committed)
{
return(committed);
}
if (_groupId != null && _memberId != null)
{
var request = new OffsetCommitRequest(_groupId, new[] { new OffsetCommitRequest.Topic(_partition.topic, _partition.partition_id, offset) }, _memberId, _generationId);
await _router.SendAsync(request, _partition.topic, _partition.partition_id, cancellationToken).ConfigureAwait(false);
}
_offsetCommitted = offset;
return(offset);
}
public async Task OffsetCommitShouldStoreAndReturnSuccess()
{
const int partitionId = 0;
var router = new BrokerRouter(_options);
await router.GetTopicMetadataAsync(TestConfig.TopicName(), CancellationToken.None);
var conn = router.GetBrokerRoute(TestConfig.TopicName(), partitionId);
// ensure the group exists
var group = new GroupCoordinatorRequest(TestConfig.ConsumerName());
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var commit = new OffsetCommitRequest(group.GroupId, new [] { new OffsetCommitRequest.Topic(TestConfig.TopicName(), partitionId, 10, null) });
var response = await conn.Connection.SendAsync(commit, CancellationToken.None);
var topic = response.Topics.FirstOrDefault();
Assert.That(topic, Is.Not.Null);
Assert.That(topic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
router.Dispose();
}
public async Task TestMultipleOffsetCommitsWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions: 2))
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var request = new OffsetCommitRequest
{
OffsetCommits =
Enumerable
.Range(0, 2)
.Select(i => new OffsetCommit {
Topic = temporaryTopic.Name,
PartitionId = i,
Offset = 0
})
.ToList()
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(2));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
Assert.That(response[1].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}
public async Task OffsetCommitShouldStoreOffsetValue()
{
const int partitionId = 0;
const long offset = 99;
using (var router = await TestConfig.IntegrationOptions.CreateRouterAsync()) {
await router.TemporaryTopicAsync(async topicName => {
await router.GetTopicMetadataAsync(topicName, CancellationToken.None);
var conn = router.GetTopicConnection(topicName, partitionId);
// ensure the group exists
var groupId = TestConfig.GroupId();
var group = new GroupCoordinatorRequest(groupId);
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.error_code, Is.EqualTo(ErrorCode.NONE));
var commit = new OffsetCommitRequest(group.group_id, new [] { new OffsetCommitRequest.Topic(topicName, partitionId, offset, null) });
var commitResponse = await conn.Connection.SendAsync(commit, CancellationToken.None);
var commitTopic = commitResponse.responses.SingleOrDefault();
Assert.That(commitTopic, Is.Not.Null);
Assert.That(commitTopic.error_code, Is.EqualTo(ErrorCode.NONE));
var fetch = new OffsetFetchRequest(groupId, new TopicPartition(topicName, partitionId));
var fetchResponse = await conn.Connection.SendAsync(fetch, CancellationToken.None);
var fetchTopic = fetchResponse.responses.SingleOrDefault();
Assert.That(fetchTopic, Is.Not.Null);
Assert.That(fetchTopic.error_code, Is.EqualTo(ErrorCode.NONE));
Assert.That(fetchTopic.offset, Is.EqualTo(offset));
});
}
}
public async Task OffsetCommitShouldStoreAndReturnSuccess()
{
const int partitionId = 0;
using (var router = await TestConfig.IntegrationOptions.CreateRouterAsync()) {
await router.TemporaryTopicAsync(async topicName => {
await router.GetTopicMetadataAsync(topicName, CancellationToken.None);
var conn = router.GetTopicConnection(topicName, partitionId);
// ensure the group exists
var groupId = TestConfig.GroupId();
var group = new GroupCoordinatorRequest(groupId);
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.error_code, Is.EqualTo(ErrorCode.NONE));
var commit = new OffsetCommitRequest(group.group_id, new [] { new OffsetCommitRequest.Topic(topicName, partitionId, 10, null) });
var response = await conn.Connection.SendAsync(commit, CancellationToken.None);
var topic = response.responses.FirstOrDefault();
Assert.That(topic, Is.Not.Null);
Assert.That(topic.error_code, Is.EqualTo(ErrorCode.NONE));
});
}
}
public void OffsetCommitRequest(
[Values(0, 1, 2)] short version,
[Values("group1", "group2")] string groupId,
[Values(0, 5)] int generation,
[Values(-1, 20000)] int retentionTime,
[Values("test", "a really long name, with spaces and punctuation!")] string topic,
[Values(1, 10)] int topicsPerRequest,
[Values(5)] int maxPartitions,
[Values(10)] int maxOffsets,
[Values(null, "something useful for the client")] string metadata)
{
var timestamp = retentionTime;
var offsetCommits = new List <OffsetCommitRequest.Topic>();
for (var t = 0; t < topicsPerRequest; t++)
{
offsetCommits.Add(new OffsetCommitRequest.Topic(
topic + t,
t % maxPartitions,
_randomizer.Next(0, int.MaxValue),
metadata,
version == 1 ? timestamp : (long?)null));
}
var request = new OffsetCommitRequest(
groupId,
offsetCommits,
version >= 1 ? "member" + generation : null,
version >= 1 ? generation : 0,
version >= 2 && retentionTime >= 0 ? (TimeSpan?)TimeSpan.FromMilliseconds(retentionTime) : null);
request.AssertCanEncodeDecodeRequest(version);
}
public ErrorCode OffsetCommit(String topic, Int32 partition, String groupId, Int64 offset)
{
EnsureLegalTopicSpelling(topic);
var broker = _topicBrokerDispatcher.SelectBroker(topic, partition);
var request = OffsetCommitRequest.CreateV0(topic, partition, groupId, offset);
var response = (OffsetCommitResponse)SubmitRequest(broker, request);
response.TryThrowFirstErrorOccured();
return(response.TopicPartitions[0].Details[0].ErrorCode);
}
/// <summary>
/// Updating the cosumerGroup's offset for the partition in topic
/// </summary>
/// <param name="consumerGroup">The consumer group</param>
/// <param name="offset">The new offset. must be larger than or equal to zero</param>
/// <returns></returns>
public async Task UpdateOrCreateOffset(string consumerGroup, long offset)
{
if (string.IsNullOrEmpty(consumerGroup))
{
throw new ArgumentNullException("consumerGroup");
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset", "offset must be positive or zero");
}
OffsetCommitRequest request = CreateOffsetCommitRequest(offset, consumerGroup);
await _gateway.SendProtocolRequest(request, _topic, _partitionId).ConfigureAwait(false);
}
private OffsetCommitRequest CreateOffsetCommitRequest(string consumerGroup, int partitionId, long offset, string metadata = null)
{
var commit = new OffsetCommitRequest
{
ConsumerGroup = consumerGroup,
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
PartitionId = partitionId,
Topic = IntegrationConfig.IntegrationTopic,
Offset = offset,
Metadata = metadata
}
}
};
return(commit);
}
private static OffsetCommitResponse OffsetCommitRequest(int partitionId, long offset, string metadata = null)
{
var request = new OffsetCommitRequest
{
ConsumerGroup = ConsumerGroup,
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
PartitionId = partitionId,
Topic = Topic,
Offset = offset,
Metadata = metadata
}
}
};
return(_brokerRoute.Connection.SendAsync(request).Result.FirstOrDefault());
}
private static void OffsetCommitRequest(int partitionId, long offset, string metadata = null)
{
var request = new OffsetCommitRequest
{
ConsumerGroup = ConsumerGroup,
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
PartitionId = partitionId,
Topic = Topic,
Offset = offset,
Metadata = metadata
}
}
};
_brokerRoute.Connection.SendAsync(request);
}
public async Task <OffsetCommitResponse> SetStoredOffsetAsync(string topic, long offset)
{
var request = new OffsetCommitRequest
{
ConsumerGroup = ConsumerGroupName,
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
PartitionId = 0,
Topic = topic,
Offset = offset
}
}
};
var result = await _router.SelectBrokerRoute(topic).Connection.SendAsync(request).ConfigureAwait(false);
return(result.FirstOrDefault());
}
private OffsetCommitRequest CreateOffsetCommitRequest(long offset, string consumerGroup)
{
OffsetCommit commit = new OffsetCommit()
{
Offset = offset,
Topic = _topic,
PartitionId = _partitionId,
TimeStamp = UseBrokerTimestamp
};
OffsetCommitRequest request = new OffsetCommitRequest()
{
ConsumerGroup = consumerGroup,
OffsetCommits = new List <OffsetCommit>()
{
commit
},
ClientId = _clientId
};
return(request);
}
public async Task TestOffsetCommitWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var request = new OffsetCommitRequest
{
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
Topic = temporaryTopic.Name,
Offset = 0
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}
public async Task OffsetCommitShouldStoreOffsetValue()
{
const int partitionId = 0;
const long offset = 99;
var router = new BrokerRouter(_options);
await router.GetTopicMetadataAsync(TestConfig.TopicName(), CancellationToken.None);
var conn = router.GetBrokerRoute(TestConfig.TopicName(), partitionId);
// ensure the group exists
var group = new GroupCoordinatorRequest(TestConfig.ConsumerName());
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var commit = new OffsetCommitRequest(group.GroupId, new [] { new OffsetCommitRequest.Topic(TestConfig.TopicName(), partitionId, offset, null) });
var commitResponse = await conn.Connection.SendAsync(commit, CancellationToken.None);
var commitTopic = commitResponse.Topics.SingleOrDefault();
Assert.That(commitTopic, Is.Not.Null);
Assert.That(commitTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var fetch = new OffsetFetchRequest(TestConfig.ConsumerName(), new TopicPartition(TestConfig.TopicName(), partitionId));
var fetchResponse = await conn.Connection.SendAsync(fetch, CancellationToken.None);
var fetchTopic = fetchResponse.Topics.SingleOrDefault();
Assert.That(fetchTopic, Is.Not.Null);
Assert.That(fetchTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
Assert.That(fetchTopic.Offset, Is.EqualTo(offset));
router.Dispose();
}
public async Task TestNewTopicProductionWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
{
var request = new MetadataRequest
{
Topics = new List <string>
{
topic
}
};
MetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Topics[0].ErrorCode == ErrorResponseCode.LeaderNotAvailable)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response, Is.Not.Null);
var first = response;
Assert.That(first.Topics, Has.Length.EqualTo(1));
var firstTopic = first.Topics.First();
Assert.That(firstTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(firstTopic.Name, Is.EqualTo(topic));
Assert.That(firstTopic.Partitions, Has.Length.EqualTo(1));
var firstPartition = firstTopic.Partitions.First();
Assert.That(firstPartition.PartitionId, Is.EqualTo(0));
}
{
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 10000,
Payload = new List <Payload>
{
new Payload
{
Topic = topic,
Partition = 0,
Codec = MessageCodec.CodecNone,
Messages = new List <Message>
{
new Message("Message 1"),
new Message("Message 2"),
new Message("Message 3"),
new Message("Message 4"),
}
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Is.Not.Null);
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 40,
Offset = 0,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(1));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(0));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 1"));
}
{
var request = new OffsetRequest
{
Offsets = new List <Offset>
{
new Offset
{
MaxOffsets = 2,
PartitionId = 0,
Time = -1,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offsets, Has.Length.EqualTo(2));
Assert.That(first.Offsets[0], Is.EqualTo(4));
Assert.That(first.Offsets[1], Is.EqualTo(0));
}
{
var request = new ConsumerMetadataRequest
{
ConsumerGroup = topic
};
ConsumerMetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Error == ErrorResponseCode.ConsumerCoordinatorNotAvailableCode)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response.Error, Is.EqualTo(ErrorResponseCode.NoError));
Console.WriteLine("Id = {0}, Host = {1}, Port = {2}", response.CoordinatorId, response.CoordinatorHost, response.CoordinatorPort);
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.Empty);
Assert.That(first.Offset, Is.EqualTo(-1));
}
{
var request = new OffsetCommitRequest
{
ConsumerGroup = topic,
ConsumerGroupGenerationId = 1,
ConsumerId = "0",
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
Metadata = "Metadata 1",
Offset = 0,
PartitionId = 0,
TimeStamp = -1,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.EqualTo("Metadata 1"));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 0 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(3));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(1));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 2"));
var lastMessage = first.Messages.Last();
Assert.That(lastMessage.Meta.Offset, Is.EqualTo(3));
Assert.That(lastMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(lastMessage.Attribute, Is.EqualTo(0));
Assert.That(lastMessage.Key, Is.Null);
Assert.That(lastMessage.MagicNumber, Is.EqualTo(0));
Assert.That(lastMessage.Value, Is.Not.Null);
var lastString = lastMessage.Value.ToUtf8String();
Assert.That(lastString, Is.EqualTo("Message 4"));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 3 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(0));
}
}
Console.WriteLine("Test completed");
}
/// <summary>
/// Get offsets for a single partitions of a given topic.
/// </summary>
/// <param name="brokerRouter">The router which provides the route and metadata.</param>
/// <param name="topicName">Name of the topic to get offset information from.</param>
/// <param name="partitionId">The partition to get offsets for.</param>
/// <param name="consumerGroup">The id of the consumer group</param>
/// <param name="offset">The new offset</param>
/// <param name="cancellationToken"></param>
public static async Task CommitTopicOffsetAsync(this IBrokerRouter brokerRouter, string topicName, int partitionId, string consumerGroup, long offset, CancellationToken cancellationToken)
{
var request = new OffsetCommitRequest(consumerGroup, new [] { new OffsetCommitRequest.Topic(topicName, partitionId, offset) });
await brokerRouter.SendAsync(request, topicName, partitionId, consumerGroup, cancellationToken).ConfigureAwait(false);
}
public async Task TestMultipleOffsetCommitsWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions:2))
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var request = new OffsetCommitRequest
{
OffsetCommits =
Enumerable
.Range(0, 2)
.Select(i => new OffsetCommit {
Topic = temporaryTopic.Name,
PartitionId = i,
Offset = 0
})
.ToList()
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(2));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
Assert.That(response[1].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}
public async Task TestNewTopicProductionWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
{
var request = new MetadataRequest
{
Topics = new List<string>
{
topic
}
};
MetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Topics[0].ErrorCode == ErrorResponseCode.LeaderNotAvailable)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response, Is.Not.Null);
var first = response;
Assert.That(first.Topics, Has.Length.EqualTo(1));
var firstTopic = first.Topics.First();
Assert.That(firstTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(firstTopic.Name, Is.EqualTo(topic));
Assert.That(firstTopic.Partitions, Has.Length.EqualTo(1));
var firstPartition = firstTopic.Partitions.First();
Assert.That(firstPartition.PartitionId, Is.EqualTo(0));
}
{
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 10000,
Payload = new List<Payload>
{
new Payload
{
Topic = topic,
Partition = 0,
Codec = MessageCodec.CodecNone,
Messages = new List<Message>
{
new Message("Message 1"),
new Message("Message 2"),
new Message("Message 3"),
new Message("Message 4"),
}
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Is.Not.Null);
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 40,
Offset = 0,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(1));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(0));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 1"));
}
{
var request = new OffsetRequest
{
Offsets = new List<Offset>
{
new Offset
{
MaxOffsets = 2,
PartitionId = 0,
Time = -1,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offsets, Has.Length.EqualTo(2));
Assert.That(first.Offsets[0], Is.EqualTo(4));
Assert.That(first.Offsets[1], Is.EqualTo(0));
}
{
var request = new ConsumerMetadataRequest
{
ConsumerGroup = topic
};
ConsumerMetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Error == ErrorResponseCode.ConsumerCoordinatorNotAvailableCode)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response.Error, Is.EqualTo(ErrorResponseCode.NoError));
Console.WriteLine("Id = {0}, Host = {1}, Port = {2}", response.CoordinatorId, response.CoordinatorHost, response.CoordinatorPort);
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.Empty);
Assert.That(first.Offset, Is.EqualTo(-1));
}
{
var request = new OffsetCommitRequest
{
ConsumerGroup = topic,
ConsumerGroupGenerationId = 1,
ConsumerId = "0",
OffsetCommits = new List<OffsetCommit>
{
new OffsetCommit
{
Metadata = "Metadata 1",
Offset = 0,
PartitionId = 0,
TimeStamp = -1,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.EqualTo("Metadata 1"));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 0 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(3));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(1));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 2"));
var lastMessage = first.Messages.Last();
Assert.That(lastMessage.Meta.Offset, Is.EqualTo(3));
Assert.That(lastMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(lastMessage.Attribute, Is.EqualTo(0));
Assert.That(lastMessage.Key, Is.Null);
Assert.That(lastMessage.MagicNumber, Is.EqualTo(0));
Assert.That(lastMessage.Value, Is.Not.Null);
var lastString = lastMessage.Value.ToUtf8String();
Assert.That(lastString, Is.EqualTo("Message 4"));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 3 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(0));
}
}
Console.WriteLine("Test completed");
}
public ValueTask <OffsetCommitResponse> OffsetCommit(OffsetCommitRequest request) => SendRequest(ApiKey.OffsetCommit, 7, request, new OffsetCommitRequestWriter(), new OffsetCommitResponseReader());
private OffsetCommitRequest CreateOffsetCommitRequest(long offset, string consumerGroup)
{
var commit = new OffsetCommit()
{
Offset = offset,
Topic = _topic,
PartitionId = _partitionId,
TimeStamp = UseBrokerTimestamp
};
var request = new OffsetCommitRequest()
{
ConsumerGroup = consumerGroup,
OffsetCommits = new List<OffsetCommit>() { commit },
ClientId = _clientId
};
return request;
}
public async Task<OffsetCommitResponse> SetStoredOffsetAsync(string topic, long offset)
{
var request = new OffsetCommitRequest
{
ConsumerGroup = ConsumerGroupName,
OffsetCommits = new List<OffsetCommit>
{
new OffsetCommit
{
PartitionId = 0,
Topic = topic,
Offset = offset
}
}
};
var result = await _router.SelectBrokerRoute(topic).Connection.SendAsync(request).ConfigureAwait(false);
return result.FirstOrDefault();
}
private OffsetCommitRequest CreateOffsetCommitRequest(string consumerGroup, int partitionId, long offset, string metadata = null)
{
var commit = new OffsetCommitRequest
{
ConsumerGroup = consumerGroup,
OffsetCommits = new List<OffsetCommit>
{
new OffsetCommit
{
PartitionId = partitionId,
Topic = IntegrationConfig.IntegrationTopic,
Offset = offset,
Metadata = metadata
}
}
};
return commit;
}
public async Task TestOffsetCommitWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var request = new OffsetCommitRequest
{
OffsetCommits = new List<OffsetCommit>
{
new OffsetCommit
{
Topic = temporaryTopic.Name,
Offset = 0
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
Assert.That(response[0].Error, Is.EqualTo(ErrorResponseCode.Unknown));
}
}