public void TestGetAllTopicMetadata()
{
// create topic
var topic1 = "testGetAllTopicMetadata1";
var topic2 = "testGetAllTopicMetadata2";
AdminUtils.CreateTopic(this.ZkClient, topic1, 1, 1, new Dictionary <string, string>());
AdminUtils.CreateTopic(this.ZkClient, topic2, 1, 1, new Dictionary <string, string>());
// wait for leader to be elected for both topics
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic1, 0, 1000);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic2, 0, 1000);
// issue metadata request with empty list of topics
var topicsMetadata =
ClientUtils.FetchTopicMetadata(
new HashSet <string>(), this.Brokers, "TopicMetadataTest-testGetAllTopicMetadata", 2000).TopicsMetadata;
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().ErrorCode);
Assert.Equal(2, topicsMetadata.Count);
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().PartitionsMetadata.First().ErrorCode);
Assert.Equal(ErrorMapping.NoError, topicsMetadata.Last().PartitionsMetadata.First().ErrorCode);
var partitionMetadataTopic1 = topicsMetadata.First().PartitionsMetadata;
var partitionMetadataTopic2 = topicsMetadata.Last().PartitionsMetadata;
Assert.Equal(1, partitionMetadataTopic1.Count);
Assert.Equal(0, partitionMetadataTopic1.First().PartitionId);
Assert.Equal(1, partitionMetadataTopic1.First().Replicas.Count());
Assert.Equal(1, partitionMetadataTopic2.Count);
Assert.Equal(0, partitionMetadataTopic2.First().PartitionId);
Assert.Equal(1, partitionMetadataTopic2.First().Replicas.Count());
}
/// <summary>
/// It updates the cache by issuing a get topic metadata request to a random broker.
/// </summary>
/// <param name="topics"></param>
/// <param name="correlationId"></param>
public void UpdateInfo(ISet <string> topics, int correlationId)
{
List <TopicMetadata> topicsMetadata;
var topicMetadataResponse = ClientUtils.FetchTopicMetadata(topics, this.brokers, this.producerConfig, correlationId);
topicsMetadata = topicMetadataResponse.TopicsMetadata;
foreach (var tmd in topicsMetadata)
{
Logger.DebugFormat("Metadata for topic {0} is {1}", tmd.Topic, tmd);
if (tmd.ErrorCode == ErrorMapping.NoError)
{
this.topicPartitionInfo[tmd.Topic] = tmd;
}
else
{
Logger.WarnFormat("Error while fetch metadata [{0}] for topic [{1}]: {2}", tmd, tmd.Topic, ErrorMapping.ExceptionFor(tmd.ErrorCode).GetType().Name);
foreach (var pmd in tmd.PartitionsMetadata)
{
if (pmd.ErrorCode != ErrorMapping.NoError &&
pmd.ErrorCode == ErrorMapping.LeaderNotAvailableCode)
{
Logger.WarnFormat("Error while fetching metadata {0} for topic partiton [{1},{2}]:[{3}]", pmd, tmd.Topic, pmd.PartitionId, ErrorMapping.ExceptionFor(pmd.ErrorCode).GetType());
//// any other error code (e.g. ReplicaNotAvailable) can be ignored since the producer does not need to access the replica and isr metadata
}
}
}
}
this.producerPool.UpdateProducer(topicsMetadata);
}
public void TestBasicTopicMetadata()
{
// create topic
var topic = "test";
AdminUtils.CreateTopic(this.ZkClient, topic, 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 1000);
var topicsMetadata = ClientUtils.FetchTopicMetadata(
new HashSet <string> {
topic
}, this.Brokers, "TopicMetadataTest-testBasicTopicMetadata", 2000)
.TopicsMetadata;
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().ErrorCode);
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().PartitionsMetadata.First().ErrorCode);
Assert.Equal(1, topicsMetadata.Count);
Assert.Equal("test", topicsMetadata.First().Topic);
var partitionMetadata = topicsMetadata.First().PartitionsMetadata;
Assert.Equal(1, partitionMetadata.Count);
Assert.Equal(0, partitionMetadata.First().PartitionId);
Assert.Equal(1, partitionMetadata.First().Replicas.Count());
}
public void TestAutoCreateTopic()
{
// auto create topic
var topic = "testAutoCreateTopic";
var topicsMetadata =
ClientUtils.FetchTopicMetadata(
new HashSet <string> {
topic
}, this.Brokers, "TopicMetadataTest-testAutoCreateTopic", 2000)
.TopicsMetadata;
Assert.Equal(ErrorMapping.LeaderNotAvailableCode, topicsMetadata.First().ErrorCode);
Assert.Equal(1, topicsMetadata.Count);
Assert.Equal(topic, topicsMetadata.First().Topic);
Assert.Equal(0, topicsMetadata.First().PartitionsMetadata.Count);
// wait for leader to be elected
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 1000);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
// retry the metadata for the auto created topic
topicsMetadata =
ClientUtils.FetchTopicMetadata(
new HashSet <string> {
topic
}, this.Brokers, "TopicMetadataTest-testBasicTopicMetadata", 2000)
.TopicsMetadata;
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().ErrorCode);
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().PartitionsMetadata.First().ErrorCode);
var partitionMetadata = topicsMetadata.First().PartitionsMetadata;
Assert.Equal(1, partitionMetadata.Count);
Assert.Equal(0, partitionMetadata.First().PartitionId);
Assert.Equal(1, partitionMetadata.First().Replicas.Count());
Assert.True(partitionMetadata.First().Leader != null);
}
public override void DoWork()
{
var leaderForPartitionsMap = new Dictionary <TopicAndPartition, Broker>();
[email protected]();
try
{
while (this.parent.NoLeaderPartitionSet.Count == 0)
{
Logger.Debug("No partition for leader election.");
this.parent.cond.Await();
}
Logger.DebugFormat("Partitions without leader {0}", string.Join(",", this.parent.NoLeaderPartitionSet));
var brokers = ZkUtils.GetAllBrokersInCluster(this.parent.zkClient);
var topicsMetadata =
ClientUtils.FetchTopicMetadata(
new HashSet <string>(this.parent.NoLeaderPartitionSet.Select(m => m.Topic)),
brokers,
this.parent.config.ClientId,
this.parent.config.SocketTimeoutMs,
this.parent.correlationId.GetAndIncrement()).TopicsMetadata;
if (Logger.IsDebugEnabled)
{
foreach (var topicMetadata in topicsMetadata)
{
Logger.Debug(topicMetadata);
}
}
foreach (var tmd in topicsMetadata)
{
var topic = tmd.Topic;
foreach (var pmd in tmd.PartitionsMetadata)
{
var topicAndPartition = new TopicAndPartition(topic, pmd.PartitionId);
if (pmd.Leader != null && this.parent.NoLeaderPartitionSet.Contains(topicAndPartition))
{
var leaderBroker = pmd.Leader;
leaderForPartitionsMap[topicAndPartition] = leaderBroker;
this.parent.NoLeaderPartitionSet.Remove(topicAndPartition);
}
}
}
}
catch (Exception e)
{
if (!isRunning.Get())
{
throw; /* If this thread is stopped, propagate this exception to kill the thread. */
}
else
{
Logger.Warn("Failed to find leader for " + string.Join(",", this.parent.NoLeaderPartitionSet), e);
}
}
finally
{
[email protected]();
}
try
{
this.parent.AddFetcherForPartitions(
leaderForPartitionsMap.ToDictionary(
kvp => kvp.Key,
kvp =>
new BrokerAndInitialOffset(kvp.Value, this.parent.partitionMap.Get(kvp.Key).GetFetchOffset())));
}
catch (Exception e)
{
if (!isRunning.Get())
{
throw; /* If this thread is stopped, propagate this exception to kill the thread. */
}
else
{
Logger.Warn(string.Format("Failed to add leader for partitions {0}; will retry", string.Join(",", leaderForPartitionsMap.Keys)), e);
[email protected]();
foreach (var leader in leaderForPartitionsMap.Keys)
{
this.parent.NoLeaderPartitionSet.Add(leader);
}
[email protected]();
}
}
this.parent.ShutdownIdleFetcherThreads();
Thread.Sleep(this.parent.config.RefreshLeaderBackoffMs);
}
