// Dispose(bool disposing) executes in two distinct scen/arios.
// If disposing equals true, the method has been called directly
// or indirectly by a user's code. Managed and unmanaged resources
// can be disposed.
// If disposing equals false, the method has been called by the
// runtime from inside the finalizer and you should not reference
// other objects. Only unmanaged resources can be disposed.
protected virtual void Dispose(bool disposing)
{
// Check to see if Dispose has already been called.
if (!this.disposed)
{
// If disposing equals true, dispose all managed
// and unmanaged resources.
if (disposing)
{
// Dispose managed resources.
this.ClearSyncProducerPool();
this.ClearLeader();
this.topic = string.Empty;
this.topicMetaRequest = null;
this.random = null;
}
// Call the appropriate methods to clean up
// unmanaged resources here.
// If disposing is false,
// only the following code is executed.
// Note disposing has been done.
this.disposed = true;
}
}
public IEnumerable <TopicMetadata> GetMetaData(TopicMetadataRequest request)
{
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
lock (this)
{
return(connection.Send(request));
}
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat("GetMetaData reconnect due to {0}", ex.FormatException());
}
}
return(null);
}
public RequestResponseSerializationTest()
{
this.producerRequest = SerializationTestUtils.CreateTestProducerRequest();
this.producerResponse = SerializationTestUtils.CreateTestProducerResponse();
this.fetchRequest = SerializationTestUtils.CreateTestFetchRequest();
this.offsetRequest = SerializationTestUtils.CreateTestOffsetRequest();
this.offsetResponse = SerializationTestUtils.CreateTestOffsetResponse();
this.topicMetadataRequest = SerializationTestUtils.CreateTestTopicMetadataRequest();
this.topicMetadataResponse = SerializationTestUtils.CreateTestTopicMetadataResponse();
}
public void TestSerializationAndDeserialization()
{
var buffer = ByteBuffer.Allocate(this.producerRequest.SizeInBytes);
this.producerRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerRequest = ProducerRequest.ReadFrom(buffer);
Assert.Equal(this.producerRequest, deserializedProducerRequest);
buffer = ByteBuffer.Allocate(this.producerResponse.SizeInBytes);
this.producerResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedProducerResponse = ProducerResponse.ReadFrom(buffer);
Assert.Equal(this.producerResponse, deserializedProducerResponse);
buffer = ByteBuffer.Allocate(this.fetchRequest.SizeInBytes);
this.fetchRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedFetchRequest = FetchRequest.ReadFrom(buffer);
Assert.Equal(this.fetchRequest, deserializedFetchRequest);
buffer = ByteBuffer.Allocate(this.offsetRequest.SizeInBytes);
this.offsetRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetRequest = OffsetRequest.ReadFrom(buffer);
Assert.Equal(this.offsetRequest, deserializedOffsetRequest);
buffer = ByteBuffer.Allocate(this.offsetResponse.SizeInBytes);
this.offsetResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedOffsetResponse = OffsetResponse.ReadFrom(buffer);
Assert.Equal(this.offsetResponse, deserializedOffsetResponse);
buffer = ByteBuffer.Allocate(this.topicMetadataRequest.SizeInBytes);
this.topicMetadataRequest.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataRequest = TopicMetadataRequest.ReadFrom(buffer);
Assert.Equal(this.topicMetadataRequest, deserializedTopicMetadataRequest);
buffer = ByteBuffer.Allocate(this.topicMetadataResponse.SizeInBytes);
this.topicMetadataResponse.WriteTo(buffer);
buffer.Rewind();
var deserializedTopicMetadataResponse = TopicMetadataResponse.ReadFrom(buffer);
Assert.Equal(this.topicMetadataResponse, deserializedTopicMetadataResponse);
}
/// <summary>
/// Used by the producer to send a metadata request since it has access to the ProducerConfig
/// </summary>
/// <param name="topics">The topics for which the metadata needs to be fetched</param>
/// <param name="brokers">The brokers in the cluster as configured on the client</param>
/// <param name="producerConfig">The producer's config</param>
/// <param name="correlationId">topic metadata response</param>
/// <returns></returns>
public static TopicMetadataResponse FetchTopicMetadata(
ISet <string> topics, IList <Broker> brokers, ProducerConfig producerConfig, int correlationId)
{
var fetchMetaDataSucceeded = false;
var i = 0;
var topicMetadataRequest = new TopicMetadataRequest(
TopicMetadataRequest.CurrentVersion, correlationId, producerConfig.ClientId, topics.ToList());
TopicMetadataResponse topicMetadataResponse = null;
Exception t = null;
// shuffle the list of brokers before sending metadata requests so that most requests don't get routed to the same broker
var shuffledBrokers = brokers.Shuffle();
while (i < shuffledBrokers.Count() && !fetchMetaDataSucceeded)
{
var producer = ProducerPool.CreateSyncProducer(producerConfig, shuffledBrokers[i]);
Logger.InfoFormat("Fetching metadata from broker {0} with correlation id {1} for {2} topic(s) {3}", shuffledBrokers[i], correlationId, topics.Count, string.Join(",", topics));
try
{
topicMetadataResponse = producer.Send(topicMetadataRequest);
fetchMetaDataSucceeded = true;
}
catch (Exception e)
{
Logger.Warn(string.Format("Fetching topic metadata with correlation id {0} for topic [{1}] from broker [{2}] failed", correlationId, topics, shuffledBrokers[i]), e);
t = e;
}
finally
{
i++;
producer.Dispose();
}
}
if (!fetchMetaDataSucceeded)
{
throw new KafkaException(
string.Format(
"fetching topic metadata for topics [{0}] from broker [{1}] failed", string.Join(",", topics), string.Join(", ", shuffledBrokers)),
t);
}
Logger.DebugFormat("Successfully fetched metadata for {0} topic(s) {1}", topics.Count(), string.Join(",", topics));
return(topicMetadataResponse);
}
// Dispose(bool disposing) executes in two distinct scen/arios.
// If disposing equals true, the method has been called directly
// or indirectly by a user's code. Managed and unmanaged resources
// can be disposed.
// If disposing equals false, the method has been called by the
// runtime from inside the finalizer and you should not reference
// other objects. Only unmanaged resources can be disposed.
protected virtual void Dispose(bool disposing)
{
// Check to see if Dispose has already been called.
if (!this.disposed)
{
// If disposing equals true, dispose all managed
// and unmanaged resources.
if (disposing)
{
// Dispose managed resources.
if (syncProducerPool != null)
{
foreach (var v in syncProducerPool)
{
if (v != null)
{
v.Dispose();
}
}
syncProducerPool = null;
}
if (this.brokerConfList != null)
{
this.brokerConfList.Clear();
this.brokerConfList = null;
}
this.topic = string.Empty;
topicMetaRequest = null;
random = null;
this.ClearLeader();
}
// Call the appropriate methods to clean up
// unmanaged resources here.
// If disposing is false,
// only the following code is executed.
// Note disposing has been done.
disposed = true;
}
}
public void TestTopicMetadataRequest()
{
// create topic
var topic = "test";
AdminUtils.CreateTopic(this.ZkClient, topic, 1, 1, new Dictionary <string, string>());
// create a topic metadata request
var topicMetadataRequest = new TopicMetadataRequest(new List <string> {
topic
}, 0);
var serializedMetadataRequest = ByteBuffer.Allocate(topicMetadataRequest.SizeInBytes + 2);
topicMetadataRequest.WriteTo(serializedMetadataRequest);
serializedMetadataRequest.Rewind();
var deserializedMetadataRequest = TopicMetadataRequest.ReadFrom(serializedMetadataRequest);
Assert.Equal(topicMetadataRequest, deserializedMetadataRequest);
}
public void TopicMetadataRequestWithNoSegmentMetadataCreation()
{
var topics = new List <string> {
"topic1", "topic2"
};
var request = TopicMetadataRequest.Create(topics, 1, 1, "test");
Assert.IsNotNull(request);
Assert.AreEqual(topics.Count, request.Topics.Count());
for (int i = 0; i < topics.Count; i++)
{
var expectedTopic = topics[i];
var actualTopic = request.Topics.ElementAt(i);
Assert.AreEqual(expectedTopic, actualTopic);
}
Assert.AreEqual(DetailedMetadataRequest.NoSegmentMetadata, request.DetailedMetadata);
}
/// <summary>
/// KafkaClientHelperWrapper produces or consumes messages to/from a particular partition of a topic
/// </summary>
/// <param name="topic">target topic</param>
/// <param name="helperConfiguration"></param>
public KafkaClientHelperWrapper(string topic, KafkaClientHelperConfiguration helperConfiguration)
{
if (string.IsNullOrEmpty(topic))
{
throw new System.ArgumentNullException("topicParam");
}
if (helperConfiguration == null)
{
throw new System.ArgumentNullException("helperConfiguration");
}
if (helperConfiguration.LeaderConfig == null && string.IsNullOrEmpty(helperConfiguration.KafkaBrokerList) && helperConfiguration.ZookeeperConfig == null)
{
throw new System.ArgumentException("Leader and KafkaBrokerList and Zookeepr connection string are missing");
}
Logger.DebugFormat("KafkaClientHelperWrapper constructor start,topicParam={0},kafkaBrokerList={1},zookeeper={2}",
topic, helperConfiguration.KafkaBrokerList, helperConfiguration.ZookeeperConfig);
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
this.topic = topic;
this.HelperConfiguration = helperConfiguration;
if (this.HelperConfiguration.LeaderConfig != null)
{
this.leaders.Add(this.HelperConfiguration.LeaderPartition, this.HelperConfiguration.LeaderConfig);
this.lastTimeLeaderFound = DateTime.Now;
}
this.topicMetaRequest = TopicMetadataRequest.Create(
new string[] { this.topic }, // topic
0, // API version id
this.random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id
}
/// <summary>
/// Force get topic metadata and update
/// </summary>
public void UpdateInfo(short versionId, int correlationId, string clientId, string topic)
{
Logger.InfoFormat("Will update metadata for topic:{0}", topic);
Guard.NotNullNorEmpty(topic, "topic");
var shuffledBrokers = this.syncProducerPool.GetShuffledProducers();
var i = 0;
var hasFetchedInfo = false;
while (i < shuffledBrokers.Count && !hasFetchedInfo)
{
ISyncProducer producer = shuffledBrokers[i++];
try
{
var topicMetadataRequest = TopicMetadataRequest.Create(new List <string>()
{
topic
}, versionId,
correlationId, clientId);
var topicMetadataList = producer.Send(topicMetadataRequest);
var topicMetadata = topicMetadataList.Any() ? topicMetadataList.First() : null;
if (topicMetadata != null)
{
if (topicMetadata.Error != ErrorMapping.NoError)
{
Logger.WarnFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, topicMetadata.Error.ToString());
}
else
{
this.topicPartitionInfo[topic] = topicMetadata;
this.topicPartitionInfoLastUpdateTime[topic] = DateTime.UtcNow;
Logger.InfoFormat("Will Update metadata info, topic {0} ", topic);
//TODO: For all partitions which has metadata, here return the sorted list.
//But sometimes kafka didn't return metadata for all topics.
this.topicPartitionInfoList[topic] = topicMetadata.PartitionsMetadata.Select(m =>
{
Partition partition = new Partition(topic, m.PartitionId);
if (m.Leader != null)
{
var leaderReplica = new Replica(m.Leader.Id, topic);
partition.Leader = leaderReplica;
Logger.InfoFormat("Topic {0} partition {1} has leader {2}", topic,
m.PartitionId, m.Leader.Id);
return(partition);
}
Logger.WarnFormat("Topic {0} partition {1} does not have a leader yet", topic,
m.PartitionId);
return(partition);
}
).OrderBy(x => x.PartId).ToList();;
hasFetchedInfo = true;
Logger.InfoFormat("Finish Update metadata info, topic {0} Partitions:{1} No leader:{2}", topic, this.topicPartitionInfoList[topic].Count, this.topicPartitionInfoList[topic].Where(r => r.Leader == null).Count());
//In very weired case, the kafka broker didn't return metadata of all broker. need break and retry. https://issues.apache.org/jira/browse/KAFKA-1998
// http://qnalist.com/questions/5899394/topicmetadata-response-miss-some-partitions-information-sometimes
if (zkClient != null)
{
Dictionary <int, int[]> topicMetaDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(this.zkClient, topic);
if (topicMetaDataInZookeeper != null && topicMetaDataInZookeeper.Any())
{
topicDataInZookeeper[topic] = topicMetaDataInZookeeper;
if (this.topicPartitionInfoList[topic].Count != topicMetaDataInZookeeper.Count)
{
Logger.ErrorFormat("NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count);
throw new UnavailableProducerException(string.Format("Please make sure every partition at least has one broker running and retry again. NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count));
}
}
}
}
}
}
catch (Exception e)
{
Logger.ErrorFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, e.FormatException());
}
}
}
/// <summary>
/// Writes a topic metadata request to the server.
/// </summary>
/// <remarks>
/// Write timeout is defaulted to infitite.
/// </remarks>
/// <param name="request">The <see cref="TopicMetadataRequest"/> to send to the server.</param>
public IEnumerable <TopicMetadata> Send(TopicMetadataRequest request)
{
this.EnsuresNotDisposed();
Guard.NotNull(request, "request");
return(this.Handle(request.RequestBuffer.GetBuffer(), new TopicMetadataRequest.Parser()));
}
public void Start()
{
while (!_stop)
{
try
{
if (_partitionsNeedingLeader.IsEmpty)
{
Thread.Sleep(_config.ConsumeGroupFindNewLeaderSleepIntervalMs);
continue;
}
PartitionTopicInfo partition;
if (_partitionsNeedingLeader.TryDequeue(out partition))
{
Logger.DebugFormat("Finding new leader for topic {0}, partition {1}", partition.Topic, partition.PartitionId);
Broker newLeader = null;
foreach (KeyValuePair <int, Broker> broker in _brokers.Brokers)
{
var consumer = new Consumer(_config, broker.Value.Host, broker.Value.Port);
try
{
IEnumerable <TopicMetadata> metaData = consumer.GetMetaData(TopicMetadataRequest.Create(new[] { partition.Topic }, 1, 0, clientId));
if (metaData != null && metaData.Any())
{
PartitionMetadata newPartitionData = metaData.First().PartitionsMetadata.FirstOrDefault(p => p.PartitionId == partition.PartitionId);
if (newPartitionData != null)
{
Logger.DebugFormat("New leader for {0} ({1}) is broker {2}", partition.Topic, partition.PartitionId, newPartitionData.Leader.Id);
newLeader = newPartitionData.Leader;
break;
}
}
}
catch (Exception ex)
{
Logger.WarnFormat("Error retrieving meta data from broker {0}: {1}", broker.Value.Id, ex.FormatException());
}
}
if (newLeader == null)
{
Logger.ErrorFormat("New leader information could not be retrieved for {0} ({1})", partition.Topic, partition.PartitionId);
}
else
{
_createNewFetcher(partition, newLeader);
}
}
}
catch (Exception ex)
{
Logger.ErrorFormat("PartitionLeaderFinder encountered an error: {0}", ex.FormatException());
Thread.Sleep(FailureRetryDelayMs);
}
}
Logger.Info("Partition leader finder thread shutting down.");
}
public TopicMetadataResponse Send(TopicMetadataRequest request)
{
var response = this.DoSend(request);
return(TopicMetadataResponse.ReadFrom(response.Buffer));
}
public IEnumerable<TopicMetadata> GetMetaData(TopicMetadataRequest request)
{
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
lock (this)
{
return connection.Send(request);
}
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat("GetMetaData reconnect due to {0}", ex.FormatException());
}
}
return null;
}
public IEnumerable<TopicMetadata> Send(TopicMetadataRequest request)
{
return this.connection.Send(request);
}
public void TopicMetadataRequestShouldThrowExceptionWhenListOfTopicsIsEmpty()
{
IList <string> topics = new List <string>();
TopicMetadataRequest.Create(topics, 1, 1, "test");
}
public KafkaClientHelper(string topicParam, int partitionIndex, KafkaClientHelperConfiguration helperConfiguration)
{
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
this.partitionIndex = partitionIndex;
this.helperConfiguration = helperConfiguration;
this.MaxMessageSize = helperConfiguration.MaxMessageSize;
this.MaxMessagePerSet = helperConfiguration.MaxMessagePerSet;
this.ConsumerMaxWait = helperConfiguration.MaxWaitTime;
this.Offset = helperConfiguration.Offset;
this.OffsetType = helperConfiguration.OffsetType;
// Logger.Debug(string.Format("KafkaClientHelper constructor start,topicParam={0},partitionIndex={1},kafkaBrokerList={2}", topicParam, partitionIndex, kafkaBrokerList));
if (string.IsNullOrEmpty(this.helperConfiguration.KafkaBrokerList))
{
throw new System.ArgumentNullException("kafkaBrokerList");
}
if (string.IsNullOrEmpty(topicParam))
{
throw new System.ArgumentNullException("topicParam");
}
this.topic = topicParam;
// assemble brokerConfig list
this.brokerConfList = new List<BrokerConfiguration>(5);
string[] brokers = this.helperConfiguration.KafkaBrokerList.Split(new char[] { ',' });
int i = 1;
foreach (string v in brokers)
{
string[] brokerParams = v.Split(new char[] { ':' });
int port = 0;
if (brokerParams.Count() == 2
&& !string.IsNullOrEmpty(brokerParams[0])
&& int.TryParse(brokerParams[1], out port))
{
this.brokerConfList.Add(new BrokerConfiguration() { BrokerId = i, Host = brokerParams[0], Port = port });
}
i++;
}
// prepare SyncProducer list
i = 0;
syncProducerPool = new SyncProducer[this.brokerConfList.Count];
this.RoundRobinSyncProducer();
topicMetaRequest = TopicMetadataRequest.Create(
new string[] { this.topic }, // topic
0, // API version id
this.random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id
}
public KafkaClientHelper(string topicParam, int partitionIndex, KafkaClientHelperConfiguration helperConfiguration)
{
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
this.partitionIndex = partitionIndex;
this.helperConfiguration = helperConfiguration;
this.MaxMessageSize = helperConfiguration.MaxMessageSize;
this.MaxMessagePerSet = helperConfiguration.MaxMessagePerSet;
this.ConsumerMaxWait = helperConfiguration.MaxWaitTime;
this.Offset = helperConfiguration.Offset;
this.OffsetType = helperConfiguration.OffsetType;
// Logger.Debug(string.Format("KafkaClientHelper constructor start,topicParam={0},partitionIndex={1},kafkaBrokerList={2}", topicParam, partitionIndex, kafkaBrokerList));
if (string.IsNullOrEmpty(this.helperConfiguration.KafkaBrokerList))
{
throw new System.ArgumentNullException("kafkaBrokerList");
}
if (string.IsNullOrEmpty(topicParam))
{
throw new System.ArgumentNullException("topicParam");
}
this.topic = topicParam;
// assemble brokerConfig list
this.brokerConfList = new List <BrokerConfiguration>(5);
string[] brokers = this.helperConfiguration.KafkaBrokerList.Split(new char[] { ',' });
int i = 1;
foreach (string v in brokers)
{
string[] brokerParams = v.Split(new char[] { ':' });
int port = 0;
if (brokerParams.Count() == 2 &&
!string.IsNullOrEmpty(brokerParams[0]) &&
int.TryParse(brokerParams[1], out port))
{
this.brokerConfList.Add(new BrokerConfiguration()
{
BrokerId = i, Host = brokerParams[0], Port = port
});
}
i++;
}
// prepare SyncProducer list
i = 0;
syncProducerPool = new SyncProducer[this.brokerConfList.Count];
this.RoundRobinSyncProducer();
topicMetaRequest = TopicMetadataRequest.Create(
new string[] { this.topic }, // topic
0, // API version id
this.random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id
}
public IEnumerable <TopicMetadata> Send(TopicMetadataRequest request)
{
return(this.connection.Send(request));
}
// Dispose(bool disposing) executes in two distinct scen/arios.
// If disposing equals true, the method has been called directly
// or indirectly by a user's code. Managed and unmanaged resources
// can be disposed.
// If disposing equals false, the method has been called by the
// runtime from inside the finalizer and you should not reference
// other objects. Only unmanaged resources can be disposed.
protected virtual void Dispose(bool disposing)
{
// Check to see if Dispose has already been called.
if (!this.disposed)
{
// If disposing equals true, dispose all managed
// and unmanaged resources.
if (disposing)
{
// Dispose managed resources.
if (syncProducerPool != null)
{
foreach (var v in syncProducerPool)
{
if (v != null)
{
v.Dispose();
}
}
syncProducerPool = null;
}
if (this.brokerConfList != null)
{
this.brokerConfList.Clear();
this.brokerConfList = null;
}
this.topic = string.Empty;
topicMetaRequest = null;
random = null;
this.ClearLeader();
}
// Call the appropriate methods to clean up
// unmanaged resources here.
// If disposing is false,
// only the following code is executed.
// Note disposing has been done.
disposed = true;
}
}