public RequestResponseSerializationTest()
{
this.producerRequest = SerializationTestUtils.CreateTestProducerRequest();
this.producerResponse = SerializationTestUtils.CreateTestProducerResponse();
this.fetchRequest = SerializationTestUtils.CreateTestFetchRequest();
this.offsetRequest = SerializationTestUtils.CreateTestOffsetRequest();
this.offsetResponse = SerializationTestUtils.CreateTestOffsetResponse();
this.topicMetadataRequest = SerializationTestUtils.CreateTestTopicMetadataRequest();
this.topicMetadataResponse = SerializationTestUtils.CreateTestTopicMetadataResponse();
}
public void TestSerializationAndDeserialization()
{
var buffer = ByteBuffer.Allocate(this.producerRequest.SizeInBytes);
this.producerRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerRequest = ProducerRequest.ReadFrom(buffer);
Assert.Equal(this.producerRequest, deserializedProducerRequest);
buffer = ByteBuffer.Allocate(this.producerResponse.SizeInBytes);
this.producerResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerResponse = ProducerResponse.ReadFrom(buffer);
Assert.Equal(this.producerResponse, deserializedProducerResponse);
buffer = ByteBuffer.Allocate(this.fetchRequest.SizeInBytes);
this.fetchRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedFetchRequest = FetchRequest.ReadFrom(buffer);
Assert.Equal(this.fetchRequest, deserializedFetchRequest);
buffer = ByteBuffer.Allocate(this.offsetRequest.SizeInBytes);
this.offsetRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetRequest = OffsetRequest.ReadFrom(buffer);
Assert.Equal(this.offsetRequest, deserializedOffsetRequest);
buffer = ByteBuffer.Allocate(this.offsetResponse.SizeInBytes);
this.offsetResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetResponse = OffsetResponse.ReadFrom(buffer);
Assert.Equal(this.offsetResponse, deserializedOffsetResponse);
buffer = ByteBuffer.Allocate(this.topicMetadataRequest.SizeInBytes);
this.topicMetadataRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataRequest = TopicMetadataRequest.ReadFrom(buffer);
Assert.Equal(this.topicMetadataRequest, deserializedTopicMetadataRequest);
buffer = ByteBuffer.Allocate(this.topicMetadataResponse.SizeInBytes);
this.topicMetadataResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataResponse = TopicMetadataResponse.ReadFrom(buffer);
Assert.Equal(this.topicMetadataResponse, deserializedTopicMetadataResponse);
}
/// <summary>
/// Used by the producer to send a metadata request since it has access to the ProducerConfig
/// </summary>
/// <param name="topics">The topics for which the metadata needs to be fetched</param>
/// <param name="brokers">The brokers in the cluster as configured on the client</param>
/// <param name="producerConfig">The producer's config</param>
/// <param name="correlationId">topic metadata response</param>
/// <returns></returns>
public static TopicMetadataResponse FetchTopicMetadata(
ISet <string> topics, IList <Broker> brokers, ProducerConfig producerConfig, int correlationId)
{
var fetchMetaDataSucceeded = false;
var i = 0;
var topicMetadataRequest = new TopicMetadataRequest(
TopicMetadataRequest.CurrentVersion, correlationId, producerConfig.ClientId, topics.ToList());
TopicMetadataResponse topicMetadataResponse = null;
Exception t = null;
// shuffle the list of brokers before sending metadata requests so that most requests don't get routed to the same broker
var shuffledBrokers = brokers.Shuffle();
while (i < shuffledBrokers.Count() && !fetchMetaDataSucceeded)
{
var producer = ProducerPool.CreateSyncProducer(producerConfig, shuffledBrokers[i]);
Logger.InfoFormat("Fetching metadata from broker {0} with correlation id {1} for {2} topic(s) {3}", shuffledBrokers[i], correlationId, topics.Count, string.Join(",", topics));
try
{
topicMetadataResponse = producer.Send(topicMetadataRequest);
fetchMetaDataSucceeded = true;
}
catch (Exception e)
{
Logger.Warn(string.Format("Fetching topic metadata with correlation id {0} for topic [{1}] from broker [{2}] failed", correlationId, topics, shuffledBrokers[i]), e);
t = e;
}
finally
{
i++;
producer.Dispose();
}
}
if (!fetchMetaDataSucceeded)
{
throw new KafkaException(
string.Format(
"fetching topic metadata for topics [{0}] from broker [{1}] failed", string.Join(",", topics), string.Join(", ", shuffledBrokers)),
t);
}
Logger.DebugFormat("Successfully fetched metadata for {0} topic(s) {1}", topics.Count(), string.Join(",", topics));
return(topicMetadataResponse);
}
public TopicMetadataResponse Send(TopicMetadataRequest request)
{
var response = this.DoSend(request);
return(TopicMetadataResponse.ReadFrom(response.Buffer));
}