public void TopicMetadataRequestWithNoSegmentMetadataCreation()
{
var topics = new List <string> {
"topic1", "topic2"
};
var request = TopicMetadataRequest.Create(topics, 1, 1, "test");
Assert.IsNotNull(request);
Assert.AreEqual(topics.Count, request.Topics.Count());
for (int i = 0; i < topics.Count; i++)
{
var expectedTopic = topics[i];
var actualTopic = request.Topics.ElementAt(i);
Assert.AreEqual(expectedTopic, actualTopic);
}
Assert.AreEqual(DetailedMetadataRequest.NoSegmentMetadata, request.DetailedMetadata);
}
/// <summary>
/// KafkaClientHelperWrapper produces or consumes messages to/from a particular partition of a topic
/// </summary>
/// <param name="topic">target topic</param>
/// <param name="helperConfiguration"></param>
public KafkaClientHelperWrapper(string topic, KafkaClientHelperConfiguration helperConfiguration)
{
if (string.IsNullOrEmpty(topic))
{
throw new System.ArgumentNullException("topicParam");
}
if (helperConfiguration == null)
{
throw new System.ArgumentNullException("helperConfiguration");
}
if (helperConfiguration.LeaderConfig == null && string.IsNullOrEmpty(helperConfiguration.KafkaBrokerList) && helperConfiguration.ZookeeperConfig == null)
{
throw new System.ArgumentException("Leader and KafkaBrokerList and Zookeepr connection string are missing");
}
Logger.DebugFormat("KafkaClientHelperWrapper constructor start,topicParam={0},kafkaBrokerList={1},zookeeper={2}",
topic, helperConfiguration.KafkaBrokerList, helperConfiguration.ZookeeperConfig);
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
this.topic = topic;
this.HelperConfiguration = helperConfiguration;
if (this.HelperConfiguration.LeaderConfig != null)
{
this.leaders.Add(this.HelperConfiguration.LeaderPartition, this.HelperConfiguration.LeaderConfig);
this.lastTimeLeaderFound = DateTime.Now;
}
this.topicMetaRequest = TopicMetadataRequest.Create(
new string[] { this.topic }, // topic
0, // API version id
this.random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id
}
public KafkaClientHelper(string topicParam, int partitionIndex, KafkaClientHelperConfiguration helperConfiguration)
{
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
this.partitionIndex = partitionIndex;
this.helperConfiguration = helperConfiguration;
this.MaxMessageSize = helperConfiguration.MaxMessageSize;
this.MaxMessagePerSet = helperConfiguration.MaxMessagePerSet;
this.ConsumerMaxWait = helperConfiguration.MaxWaitTime;
this.Offset = helperConfiguration.Offset;
this.OffsetType = helperConfiguration.OffsetType;
// Logger.Debug(string.Format("KafkaClientHelper constructor start,topicParam={0},partitionIndex={1},kafkaBrokerList={2}", topicParam, partitionIndex, kafkaBrokerList));
if (string.IsNullOrEmpty(this.helperConfiguration.KafkaBrokerList))
{
throw new System.ArgumentNullException("kafkaBrokerList");
}
if (string.IsNullOrEmpty(topicParam))
{
throw new System.ArgumentNullException("topicParam");
}
this.topic = topicParam;
// assemble brokerConfig list
this.brokerConfList = new List <BrokerConfiguration>(5);
string[] brokers = this.helperConfiguration.KafkaBrokerList.Split(new char[] { ',' });
int i = 1;
foreach (string v in brokers)
{
string[] brokerParams = v.Split(new char[] { ':' });
int port = 0;
if (brokerParams.Count() == 2 &&
!string.IsNullOrEmpty(brokerParams[0]) &&
int.TryParse(brokerParams[1], out port))
{
this.brokerConfList.Add(new BrokerConfiguration()
{
BrokerId = i, Host = brokerParams[0], Port = port
});
}
i++;
}
// prepare SyncProducer list
i = 0;
syncProducerPool = new SyncProducer[this.brokerConfList.Count];
this.RoundRobinSyncProducer();
topicMetaRequest = TopicMetadataRequest.Create(
new string[] { this.topic }, // topic
0, // API version id
this.random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id
}
/// <summary>
/// Force get topic metadata and update
/// </summary>
public void UpdateInfo(short versionId, int correlationId, string clientId, string topic)
{
Logger.InfoFormat("Will update metadata for topic:{0}", topic);
Guard.NotNullNorEmpty(topic, "topic");
var shuffledBrokers = this.syncProducerPool.GetShuffledProducers();
var i = 0;
var hasFetchedInfo = false;
while (i < shuffledBrokers.Count && !hasFetchedInfo)
{
ISyncProducer producer = shuffledBrokers[i++];
try
{
var topicMetadataRequest = TopicMetadataRequest.Create(new List <string>()
{
topic
}, versionId,
correlationId, clientId);
var topicMetadataList = producer.Send(topicMetadataRequest);
var topicMetadata = topicMetadataList.Any() ? topicMetadataList.First() : null;
if (topicMetadata != null)
{
if (topicMetadata.Error != ErrorMapping.NoError)
{
Logger.WarnFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, topicMetadata.Error.ToString());
}
else
{
this.topicPartitionInfo[topic] = topicMetadata;
this.topicPartitionInfoLastUpdateTime[topic] = DateTime.UtcNow;
Logger.InfoFormat("Will Update metadata info, topic {0} ", topic);
//TODO: For all partitions which has metadata, here return the sorted list.
//But sometimes kafka didn't return metadata for all topics.
this.topicPartitionInfoList[topic] = topicMetadata.PartitionsMetadata.Select(m =>
{
Partition partition = new Partition(topic, m.PartitionId);
if (m.Leader != null)
{
var leaderReplica = new Replica(m.Leader.Id, topic);
partition.Leader = leaderReplica;
Logger.InfoFormat("Topic {0} partition {1} has leader {2}", topic,
m.PartitionId, m.Leader.Id);
return(partition);
}
Logger.WarnFormat("Topic {0} partition {1} does not have a leader yet", topic,
m.PartitionId);
return(partition);
}
).OrderBy(x => x.PartId).ToList();;
hasFetchedInfo = true;
Logger.InfoFormat("Finish Update metadata info, topic {0} Partitions:{1} No leader:{2}", topic, this.topicPartitionInfoList[topic].Count, this.topicPartitionInfoList[topic].Where(r => r.Leader == null).Count());
//In very weired case, the kafka broker didn't return metadata of all broker. need break and retry. https://issues.apache.org/jira/browse/KAFKA-1998
// http://qnalist.com/questions/5899394/topicmetadata-response-miss-some-partitions-information-sometimes
if (zkClient != null)
{
Dictionary <int, int[]> topicMetaDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(this.zkClient, topic);
if (topicMetaDataInZookeeper != null && topicMetaDataInZookeeper.Any())
{
topicDataInZookeeper[topic] = topicMetaDataInZookeeper;
if (this.topicPartitionInfoList[topic].Count != topicMetaDataInZookeeper.Count)
{
Logger.ErrorFormat("NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count);
throw new UnavailableProducerException(string.Format("Please make sure every partition at least has one broker running and retry again. NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count));
}
}
}
}
}
}
catch (Exception e)
{
Logger.ErrorFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, e.FormatException());
}
}
}
public void Start()
{
while (!_stop)
{
try
{
if (_partitionsNeedingLeader.IsEmpty)
{
Thread.Sleep(_config.ConsumeGroupFindNewLeaderSleepIntervalMs);
continue;
}
PartitionTopicInfo partition;
if (_partitionsNeedingLeader.TryDequeue(out partition))
{
Logger.DebugFormat("Finding new leader for topic {0}, partition {1}", partition.Topic, partition.PartitionId);
Broker newLeader = null;
foreach (KeyValuePair <int, Broker> broker in _brokers.Brokers)
{
var consumer = new Consumer(_config, broker.Value.Host, broker.Value.Port);
try
{
IEnumerable <TopicMetadata> metaData = consumer.GetMetaData(TopicMetadataRequest.Create(new[] { partition.Topic }, 1, 0, clientId));
if (metaData != null && metaData.Any())
{
PartitionMetadata newPartitionData = metaData.First().PartitionsMetadata.FirstOrDefault(p => p.PartitionId == partition.PartitionId);
if (newPartitionData != null)
{
Logger.DebugFormat("New leader for {0} ({1}) is broker {2}", partition.Topic, partition.PartitionId, newPartitionData.Leader.Id);
newLeader = newPartitionData.Leader;
break;
}
}
}
catch (Exception ex)
{
Logger.WarnFormat("Error retrieving meta data from broker {0}: {1}", broker.Value.Id, ex.FormatException());
}
}
if (newLeader == null)
{
Logger.ErrorFormat("New leader information could not be retrieved for {0} ({1})", partition.Topic, partition.PartitionId);
}
else
{
_createNewFetcher(partition, newLeader);
}
}
}
catch (Exception ex)
{
Logger.ErrorFormat("PartitionLeaderFinder encountered an error: {0}", ex.FormatException());
Thread.Sleep(FailureRetryDelayMs);
}
}
Logger.Info("Partition leader finder thread shutting down.");
}
public void TopicMetadataRequestShouldThrowExceptionWhenListOfTopicsIsEmpty()
{
IList <string> topics = new List <string>();
TopicMetadataRequest.Create(topics, 1, 1, "test");
}