public async Task <Dictionary <string, int> > BuildOffsetCoordinatorMapAsync(CancellationToken token, params string[] consumerGroups)
{
if (connections.Count == 0)
{
await RefreshAsync(token).ConfigureAwait(false);
}
foreach (var consumerGroup in consumerGroups)
{
var currentCoordinator = offsetCoordinatorMap.GetOrCreate(consumerGroup, () => - 1);
if (currentCoordinator == -1)
{
var request = new ConsumerMetadataRequest {
ConsumerGroup = consumerGroup
};
var response = await connections.Values.First().SendRequestAsync(request, token).ConfigureAwait(false);
if (response.Error != ErrorResponseCode.NoError)
{
throw new InvalidOperationException("Failed to retrieve consumer offsets " + response.Error);
}
offsetCoordinatorMap[consumerGroup] = response.CoordinatorId;
}
}
return(offsetCoordinatorMap);
}
private static ConsumerMetadataResponse ConsumerMetadataRequest()
{
var request = new ConsumerMetadataRequest {
ConsumerGroup = ConsumerGroup
};
return(_brokerRoute.Connection.SendAsync(request).Result.FirstOrDefault());
}
public void ConsumerMetadataRequestShouldReturnWithoutError()
{
using (var router = new BrokerRouter(Options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic);
var request = new ConsumerMetadataRequest {
ConsumerGroup = IntegrationConfig.IntegrationConsumer
};
var response = conn.Connection.SendAsync(request).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
}
}
public async Task TestNewTopicProductionWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
{
var request = new MetadataRequest
{
Topics = new List <string>
{
topic
}
};
MetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Topics[0].ErrorCode == ErrorResponseCode.LeaderNotAvailable)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response, Is.Not.Null);
var first = response;
Assert.That(first.Topics, Has.Length.EqualTo(1));
var firstTopic = first.Topics.First();
Assert.That(firstTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(firstTopic.Name, Is.EqualTo(topic));
Assert.That(firstTopic.Partitions, Has.Length.EqualTo(1));
var firstPartition = firstTopic.Partitions.First();
Assert.That(firstPartition.PartitionId, Is.EqualTo(0));
}
{
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 10000,
Payload = new List <Payload>
{
new Payload
{
Topic = topic,
Partition = 0,
Codec = MessageCodec.CodecNone,
Messages = new List <Message>
{
new Message("Message 1"),
new Message("Message 2"),
new Message("Message 3"),
new Message("Message 4"),
}
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Is.Not.Null);
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 40,
Offset = 0,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(1));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(0));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 1"));
}
{
var request = new OffsetRequest
{
Offsets = new List <Offset>
{
new Offset
{
MaxOffsets = 2,
PartitionId = 0,
Time = -1,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offsets, Has.Length.EqualTo(2));
Assert.That(first.Offsets[0], Is.EqualTo(4));
Assert.That(first.Offsets[1], Is.EqualTo(0));
}
{
var request = new ConsumerMetadataRequest
{
ConsumerGroup = topic
};
ConsumerMetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Error == ErrorResponseCode.ConsumerCoordinatorNotAvailableCode)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response.Error, Is.EqualTo(ErrorResponseCode.NoError));
Console.WriteLine("Id = {0}, Host = {1}, Port = {2}", response.CoordinatorId, response.CoordinatorHost, response.CoordinatorPort);
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.Empty);
Assert.That(first.Offset, Is.EqualTo(-1));
}
{
var request = new OffsetCommitRequest
{
ConsumerGroup = topic,
ConsumerGroupGenerationId = 1,
ConsumerId = "0",
OffsetCommits = new List <OffsetCommit>
{
new OffsetCommit
{
Metadata = "Metadata 1",
Offset = 0,
PartitionId = 0,
TimeStamp = -1,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List <OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.EqualTo("Metadata 1"));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 0 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(3));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(1));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 2"));
var lastMessage = first.Messages.Last();
Assert.That(lastMessage.Meta.Offset, Is.EqualTo(3));
Assert.That(lastMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(lastMessage.Attribute, Is.EqualTo(0));
Assert.That(lastMessage.Key, Is.Null);
Assert.That(lastMessage.MagicNumber, Is.EqualTo(0));
Assert.That(lastMessage.Value, Is.Not.Null);
var lastString = lastMessage.Value.ToUtf8String();
Assert.That(lastString, Is.EqualTo("Message 4"));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List <Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 3 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(0));
}
}
Console.WriteLine("Test completed");
}
public void ConsumerMetadataRequestShouldReturnWithoutError()
{
using (var router = new BrokerRouter(Options))
{
var conn = router.SelectBrokerRoute(IntegrationConfig.IntegrationTopic);
var request = new ConsumerMetadataRequest {ConsumerGroup = IntegrationConfig.IntegrationConsumer};
var response = conn.Connection.SendAsync(request).Result.FirstOrDefault();
Assert.That(response, Is.Not.Null);
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
}
}
public async Task TestNewTopicProductionWorksOk()
{
using (var temporaryTopic = testCluster.CreateTemporaryTopic())
using (var connection = await KafkaConnectionFactory.CreateSimpleKafkaConnectionAsync(testCluster.CreateBrokerUris()[0]))
{
var topic = temporaryTopic.Name;
{
var request = new MetadataRequest
{
Topics = new List<string>
{
topic
}
};
MetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Topics[0].ErrorCode == ErrorResponseCode.LeaderNotAvailable)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response, Is.Not.Null);
var first = response;
Assert.That(first.Topics, Has.Length.EqualTo(1));
var firstTopic = first.Topics.First();
Assert.That(firstTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(firstTopic.Name, Is.EqualTo(topic));
Assert.That(firstTopic.Partitions, Has.Length.EqualTo(1));
var firstPartition = firstTopic.Partitions.First();
Assert.That(firstPartition.PartitionId, Is.EqualTo(0));
}
{
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 10000,
Payload = new List<Payload>
{
new Payload
{
Topic = topic,
Partition = 0,
Codec = MessageCodec.CodecNone,
Messages = new List<Message>
{
new Message("Message 1"),
new Message("Message 2"),
new Message("Message 3"),
new Message("Message 4"),
}
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Is.Not.Null);
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 40,
Offset = 0,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(1));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(0));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 1"));
}
{
var request = new OffsetRequest
{
Offsets = new List<Offset>
{
new Offset
{
MaxOffsets = 2,
PartitionId = 0,
Time = -1,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Offsets, Has.Length.EqualTo(2));
Assert.That(first.Offsets[0], Is.EqualTo(4));
Assert.That(first.Offsets[1], Is.EqualTo(0));
}
{
var request = new ConsumerMetadataRequest
{
ConsumerGroup = topic
};
ConsumerMetadataResponse response = null;
while (response == null)
{
response = await connection.SendRequestAsync(request, CancellationToken.None);
if (response.Error == ErrorResponseCode.ConsumerCoordinatorNotAvailableCode)
{
response = null;
await Task.Delay(1000);
}
}
Assert.That(response.Error, Is.EqualTo(ErrorResponseCode.NoError));
Console.WriteLine("Id = {0}, Host = {1}, Port = {2}", response.CoordinatorId, response.CoordinatorHost, response.CoordinatorPort);
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.Empty);
Assert.That(first.Offset, Is.EqualTo(-1));
}
{
var request = new OffsetCommitRequest
{
ConsumerGroup = topic,
ConsumerGroupGenerationId = 1,
ConsumerId = "0",
OffsetCommits = new List<OffsetCommit>
{
new OffsetCommit
{
Metadata = "Metadata 1",
Offset = 0,
PartitionId = 0,
TimeStamp = -1,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
}
{
var request = new OffsetFetchRequest
{
ConsumerGroup = topic,
Topics = new List<OffsetFetch>
{
new OffsetFetch
{
PartitionId = 0,
Topic = topic
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.MetaData, Is.EqualTo("Metadata 1"));
Assert.That(first.Offset, Is.EqualTo(0));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 0 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(3));
var firstMessage = first.Messages.First();
Assert.That(firstMessage.Meta.Offset, Is.EqualTo(1));
Assert.That(firstMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(firstMessage.Attribute, Is.EqualTo(0));
Assert.That(firstMessage.Key, Is.Null);
Assert.That(firstMessage.MagicNumber, Is.EqualTo(0));
Assert.That(firstMessage.Value, Is.Not.Null);
var firstString = firstMessage.Value.ToUtf8String();
Assert.That(firstString, Is.EqualTo("Message 2"));
var lastMessage = first.Messages.Last();
Assert.That(lastMessage.Meta.Offset, Is.EqualTo(3));
Assert.That(lastMessage.Meta.PartitionId, Is.EqualTo(0));
Assert.That(lastMessage.Attribute, Is.EqualTo(0));
Assert.That(lastMessage.Key, Is.Null);
Assert.That(lastMessage.MagicNumber, Is.EqualTo(0));
Assert.That(lastMessage.Value, Is.Not.Null);
var lastString = lastMessage.Value.ToUtf8String();
Assert.That(lastString, Is.EqualTo("Message 4"));
}
{
var request = new FetchRequest
{
MinBytes = 0,
MaxWaitTime = 0,
Fetches = new List<Fetch>
{
new Fetch
{
MaxBytes = 1024,
Offset = 3 + 1,
PartitionId = 0,
Topic = topic,
}
}
};
var response = await connection.SendRequestAsync(request, CancellationToken.None);
Assert.That(response, Has.Count.EqualTo(1));
var first = response.First();
Assert.That(first.Error, Is.EqualTo(ErrorResponseCode.NoError));
Assert.That(first.HighWaterMark, Is.EqualTo(4));
Assert.That(first.PartitionId, Is.EqualTo(0));
Assert.That(first.Topic, Is.EqualTo(topic));
Assert.That(first.Messages, Has.Count.EqualTo(0));
}
}
Console.WriteLine("Test completed");
}
