public Partition(string topic, int partId, Replica leader = null, HashSet<Replica> assignedReplicas = null,
HashSet<Replica> inSyncReplica = null, HashSet<Replica> catchUpReplicas = null, HashSet<Replica> reassignedReplicas = null)
{
this.Topic = topic;
this.Leader = leader;
this.PartId = partId;
this.AssignedReplicas = assignedReplicas ?? new HashSet<Replica>();
this.InSyncReplicas = inSyncReplica ?? new HashSet<Replica>();
this.CatchUpReplicas = catchUpReplicas ?? new HashSet<Replica>();
this.ReassignedReplicas = reassignedReplicas ?? new HashSet<Replica>();
}
public Partition(string topic,
int partId,
Replica leader = null,
HashSet <Replica> assignedReplicas = null,
HashSet <Replica> inSyncReplica = null,
HashSet <Replica> catchUpReplicas = null,
HashSet <Replica> reassignedReplicas = null)
{
Topic = topic;
Leader = leader;
PartId = partId;
AssignedReplicas = assignedReplicas ?? new HashSet <Replica>();
InSyncReplicas = inSyncReplica ?? new HashSet <Replica>();
CatchUpReplicas = catchUpReplicas ?? new HashSet <Replica>();
ReassignedReplicas = reassignedReplicas ?? new HashSet <Replica>();
}
public void ShouldHandleEvents()
{
var partitioner = new Mock<IPartitioner<string>>();
var config = new ProducerConfiguration(new List<BrokerConfiguration>());
var pool = new Mock<ISyncProducerPool>();
var producer = new Mock<ISyncProducer>();
var partitionMetadatas = new List<PartitionMetadata>()
{
new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty<Broker>(),
Enumerable.Empty<Broker>())
};
var metadatas = new List<TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.NoError) };
producer.SetupGet(p => p.Config)
.Returns(
() =>
new SyncProducerConfiguration(new ProducerConfiguration(new List<BrokerConfiguration>()), 0,
"host1", 1234));
producer.Setup(p => p.Send(It.IsAny<TopicMetadataRequest>())).Returns(() => metadatas);
var statuses = new Dictionary<TopicAndPartition, ProducerResponseStatus>();
statuses[new TopicAndPartition("test", 0)] = new ProducerResponseStatus();
producer.Setup(p => p.Send(It.IsAny<ProducerRequest>()))
.Returns(
() =>
new ProducerResponse(1, statuses));
pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List<ISyncProducer>() { producer.Object });
pool.Setup(p => p.GetProducer(It.IsAny<int>())).Returns(() => producer.Object);
var mockPartitionInfo = new Mock<IBrokerPartitionInfo>();
mockPartitionInfo.Setup(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test"))
.Returns(() =>
{
var partition = new Partition("test", 0);
var replica = new Replica(0, "test");
partition.Leader = replica;
return new List<Partition>() { partition };
});
var handler = new DefaultCallbackHandler<string, Message>(config, partitioner.Object, new DefaultEncoder(), mockPartitionInfo.Object, pool.Object);
handler.Handle(new List<ProducerData<string, Message>>() { new ProducerData<string, Message>("test", new Message(new byte[100])) });
pool.Verify(p => p.GetProducer(0));
}
/// <summary>
/// Force get topic metadata and update
/// </summary>
public void UpdateInfo(short versionId, int correlationId, string clientId, string topic)
{
Logger.InfoFormat("Will update metadata for topic:{0}", topic);
Guard.NotNullNorEmpty(topic, "topic");
var shuffledBrokers = this.syncProducerPool.GetShuffledProducers();
var i = 0;
var hasFetchedInfo = false;
while (i < shuffledBrokers.Count && !hasFetchedInfo)
{
ISyncProducer producer = shuffledBrokers[i++];
try
{
var topicMetadataRequest = TopicMetadataRequest.Create(new List<string>() { topic }, versionId,
correlationId, clientId);
var topicMetadataList = producer.Send(topicMetadataRequest);
var topicMetadata = topicMetadataList.Any() ? topicMetadataList.First() : null;
if (topicMetadata != null)
{
if (topicMetadata.Error != ErrorMapping.NoError)
{
Logger.WarnFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, topicMetadata.Error.ToString());
}
else
{
this.topicPartitionInfo[topic] = topicMetadata;
this.topicPartitionInfoLastUpdateTime[topic] = DateTime.UtcNow;
Logger.InfoFormat("Will Update metadata info, topic {0} ", topic);
//TODO: For all partitions which has metadata, here return the sorted list.
//But sometimes kafka didn't return metadata for all topics.
this.topicPartitionInfoList[topic] = topicMetadata.PartitionsMetadata.Select(m =>
{
Partition partition = new Partition(topic, m.PartitionId);
if (m.Leader != null)
{
var leaderReplica = new Replica(m.Leader.Id, topic);
partition.Leader = leaderReplica;
Logger.InfoFormat("Topic {0} partition {1} has leader {2}", topic,
m.PartitionId, m.Leader.Id);
return partition;
}
Logger.WarnFormat("Topic {0} partition {1} does not have a leader yet", topic,
m.PartitionId);
return partition;
}
).OrderBy(x => x.PartId).ToList(); ;
hasFetchedInfo = true;
Logger.InfoFormat("Finish Update metadata info, topic {0} Partitions:{1} No leader:{2}", topic, this.topicPartitionInfoList[topic].Count, this.topicPartitionInfoList[topic].Where(r => r.Leader == null).Count());
//In very weired case, the kafka broker didn't return metadata of all broker. need break and retry. https://issues.apache.org/jira/browse/KAFKA-1998
// http://qnalist.com/questions/5899394/topicmetadata-response-miss-some-partitions-information-sometimes
if (zkClient != null)
{
Dictionary<int, int[]> topicMetaDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(this.zkClient, topic);
if (topicMetaDataInZookeeper != null && topicMetaDataInZookeeper.Any())
{
topicDataInZookeeper[topic] = topicMetaDataInZookeeper;
if (this.topicPartitionInfoList[topic].Count != topicMetaDataInZookeeper.Count)
{
Logger.ErrorFormat("NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count);
throw new UnavailableProducerException(string.Format("Please make sure every partition at least has one broker running and retry again. NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count));
}
}
}
}
}
}
catch (Exception e)
{
Logger.ErrorFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, e.FormatException());
}
}
}