public void ShouldHandleEvents()
{
var partitioner = new Mock<IPartitioner<string>>();
var config = new ProducerConfiguration(new List<BrokerConfiguration>());
var pool = new Mock<ISyncProducerPool>();
var producer = new Mock<ISyncProducer>();
var partitionMetadatas = new List<PartitionMetadata>()
{
new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty<Broker>(),
Enumerable.Empty<Broker>())
};
var metadatas = new List<TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.NoError) };
producer.SetupGet(p => p.Config)
.Returns(
() =>
new SyncProducerConfiguration(new ProducerConfiguration(new List<BrokerConfiguration>()), 0,
"host1", 1234));
producer.Setup(p => p.Send(It.IsAny<TopicMetadataRequest>())).Returns(() => metadatas);
var statuses = new Dictionary<TopicAndPartition, ProducerResponseStatus>();
statuses[new TopicAndPartition("test", 0)] = new ProducerResponseStatus();
producer.Setup(p => p.Send(It.IsAny<ProducerRequest>()))
.Returns(
() =>
new ProducerResponse(1, statuses));
pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List<ISyncProducer>() { producer.Object });
pool.Setup(p => p.GetProducer(It.IsAny<int>())).Returns(() => producer.Object);
var mockPartitionInfo = new Mock<IBrokerPartitionInfo>();
mockPartitionInfo.Setup(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test"))
.Returns(() =>
{
var partition = new Partition("test", 0);
var replica = new Replica(0, "test");
partition.Leader = replica;
return new List<Partition>() { partition };
});
var handler = new DefaultCallbackHandler<string, Message>(config, partitioner.Object, new DefaultEncoder(), mockPartitionInfo.Object, pool.Object);
handler.Handle(new List<ProducerData<string, Message>>() { new ProducerData<string, Message>("test", new Message(new byte[100])) });
pool.Verify(p => p.GetProducer(0));
}
/// <summary>
/// Initializes a new instance of the <see cref="PartitionTopicInfo"/> class.
/// </summary>
/// <param name="topic">
/// The topic.
/// </param>
/// <param name="brokerId">
/// The broker ID.
/// </param>
/// <param name="partition">
/// The broker's partition.
/// </param>
/// <param name="chunkQueue">
/// The chunk queue.
/// </param>
/// <param name="consumedOffset">
/// The consumed offset value.
/// </param>
/// <param name="fetchedOffset">
/// The fetched offset value.
/// </param>
/// <param name="fetchSize">
/// The fetch size.
/// </param>
public PartitionTopicInfo(
string topic,
int brokerId,
Partition partition,
BlockingCollection<FetchedDataChunk> chunkQueue,
long consumedOffset,
long fetchedOffset,
int fetchSize)
{
this.Topic = topic;
this.Partition = partition;
this.chunkQueue = chunkQueue;
this.BrokerId = brokerId;
this.consumedOffset = consumedOffset;
this.fetchedOffset = fetchedOffset;
this.FetchSize = fetchSize;
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(
CultureInfo.CurrentCulture, "initial consumer offset of {0} is {1}", this, consumedOffset);
Logger.DebugFormat(
CultureInfo.CurrentCulture, "initial fetch offset of {0} is {1}", this, fetchedOffset);
}
}
/// <summary>
/// Initializes the topic - broker's partitions mappings.
/// </summary>
private void InitializeTopicBrokerPartitions()
{
if (this.topicBrokerPartitions != null)
{
return;
}
this.topicBrokerPartitions = new Dictionary<string, SortedSet<Partition>>();
this.zkclient.MakeSurePersistentPathExists(ZooKeeperClient.DefaultBrokerTopicsPath);
IList<string> topics = this.zkclient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerTopicsPath);
foreach (string topic in topics)
{
string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
IList<string> brokersPerTopic = this.zkclient.GetChildrenParentMayNotExist(brokerTopicPath);
var brokerPartitions = new SortedDictionary<int, int>();
foreach (string brokerId in brokersPerTopic)
{
string path = brokerTopicPath + "/" + brokerId;
var numPartitionsPerBrokerAndTopic = this.zkclient.ReadData<string>(path);
brokerPartitions.Add(int.Parse(brokerId, CultureInfo.InvariantCulture), int.Parse(numPartitionsPerBrokerAndTopic, CultureInfo.CurrentCulture));
}
var brokerParts = new SortedSet<Partition>();
foreach (var brokerPartition in brokerPartitions)
{
for (int i = 0; i < brokerPartition.Value; i++)
{
var bidPid = new Partition(brokerPartition.Key, i);
brokerParts.Add(bidPid);
}
}
this.topicBrokerPartitions.Add(topic, brokerParts);
}
}
private long ResetConsumerOffsets(string topic, Partition partition)
{
long offset;
switch (this.config.AutoOffsetReset)
{
case OffsetRequest.SmallestTime:
offset = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTime:
offset = OffsetRequest.LatestTime;
break;
default:
return -1;
}
var request = new OffsetRequest(topic, partition.PartId, offset, 1);
var offsets = this.simpleConsumer.GetOffsetsBefore(request);
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic);
Logger.InfoFormat(CultureInfo.CurrentCulture, "updating partition {0} with {1} offset {2}", partition.Name, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsets[0]);
ZkUtils.UpdatePersistentPath(this.zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Name, offsets[0].ToString());
return offsets[0];
}
/// <summary>
/// Generate the updated mapping of (brokerId, numPartitions) for the new list of brokers
/// registered under some topic.
/// </summary>
/// <param name="topic">The path of the topic under which the brokers have changed..</param>
/// <param name="childs">The list of changed brokers.</param>
private void ProcessNewBrokerInExistingTopic(string topic, IEnumerable<string> childs)
{
if (this.actualBrokerTopicsPartitionsMap.ContainsKey(topic))
{
Logger.Debug("Old list of brokers -> " + this.oldBrokerTopicsPartitionsMap[topic].ToMultiString(x => x.BrokerId.ToString(), ","));
}
var updatedBrokers = new SortedSet<int>(childs.Select(x => int.Parse(x, CultureInfo.InvariantCulture)));
string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
var sortedBrokerPartitions = new SortedDictionary<int, int>();
foreach (var bid in updatedBrokers)
{
var num = this.zkclient.ReadData<string>(brokerTopicPath + "/" + bid);
sortedBrokerPartitions.Add(bid, int.Parse(num, CultureInfo.InvariantCulture));
}
var updatedBrokerParts = new SortedSet<Partition>();
foreach (var bp in sortedBrokerPartitions)
{
for (int i = 0; i < bp.Value; i++)
{
var bidPid = new Partition(bp.Key, i);
updatedBrokerParts.Add(bidPid);
}
}
Logger.Debug("Currently registered list of brokers for topic " + topic + " -> " + childs.ToMultiString(", "));
SortedSet<Partition> mergedBrokerParts = updatedBrokerParts;
if (this.actualBrokerTopicsPartitionsMap.ContainsKey(topic))
{
SortedSet<Partition> oldBrokerParts = this.actualBrokerTopicsPartitionsMap[topic];
Logger.Debug(
"Unregistered list of brokers for topic " + topic + " -> " + oldBrokerParts.ToMultiString(", "));
foreach (var oldBrokerPart in oldBrokerParts)
{
mergedBrokerParts.Add(oldBrokerPart);
}
}
else
{
this.actualBrokerTopicsPartitionsMap.Add(topic, null);
}
this.actualBrokerTopicsPartitionsMap[topic] = new SortedSet<Partition>(mergedBrokerParts.Where(x => this.actualBrokerIdMap.ContainsKey(x.BrokerId)));
}
/// <summary>
/// Force get topic metadata and update
/// </summary>
public void UpdateInfo(short versionId, int correlationId, string clientId, string topic)
{
Logger.InfoFormat("Will update metadata for topic:{0}", topic);
Guard.NotNullNorEmpty(topic, "topic");
var shuffledBrokers = this.syncProducerPool.GetShuffledProducers();
var i = 0;
var hasFetchedInfo = false;
while (i < shuffledBrokers.Count && !hasFetchedInfo)
{
ISyncProducer producer = shuffledBrokers[i++];
try
{
var topicMetadataRequest = TopicMetadataRequest.Create(new List<string>() { topic }, versionId,
correlationId, clientId);
var topicMetadataList = producer.Send(topicMetadataRequest);
var topicMetadata = topicMetadataList.Any() ? topicMetadataList.First() : null;
if (topicMetadata != null)
{
if (topicMetadata.Error != ErrorMapping.NoError)
{
Logger.WarnFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, topicMetadata.Error.ToString());
}
else
{
this.topicPartitionInfo[topic] = topicMetadata;
this.topicPartitionInfoLastUpdateTime[topic] = DateTime.UtcNow;
Logger.InfoFormat("Will Update metadata info, topic {0} ", topic);
//TODO: For all partitions which has metadata, here return the sorted list.
//But sometimes kafka didn't return metadata for all topics.
this.topicPartitionInfoList[topic] = topicMetadata.PartitionsMetadata.Select(m =>
{
Partition partition = new Partition(topic, m.PartitionId);
if (m.Leader != null)
{
var leaderReplica = new Replica(m.Leader.Id, topic);
partition.Leader = leaderReplica;
Logger.InfoFormat("Topic {0} partition {1} has leader {2}", topic,
m.PartitionId, m.Leader.Id);
return partition;
}
Logger.WarnFormat("Topic {0} partition {1} does not have a leader yet", topic,
m.PartitionId);
return partition;
}
).OrderBy(x => x.PartId).ToList(); ;
hasFetchedInfo = true;
Logger.InfoFormat("Finish Update metadata info, topic {0} Partitions:{1} No leader:{2}", topic, this.topicPartitionInfoList[topic].Count, this.topicPartitionInfoList[topic].Where(r => r.Leader == null).Count());
//In very weired case, the kafka broker didn't return metadata of all broker. need break and retry. https://issues.apache.org/jira/browse/KAFKA-1998
// http://qnalist.com/questions/5899394/topicmetadata-response-miss-some-partitions-information-sometimes
if (zkClient != null)
{
Dictionary<int, int[]> topicMetaDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(this.zkClient, topic);
if (topicMetaDataInZookeeper != null && topicMetaDataInZookeeper.Any())
{
topicDataInZookeeper[topic] = topicMetaDataInZookeeper;
if (this.topicPartitionInfoList[topic].Count != topicMetaDataInZookeeper.Count)
{
Logger.ErrorFormat("NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count);
throw new UnavailableProducerException(string.Format("Please make sure every partition at least has one broker running and retry again. NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count));
}
}
}
}
}
}
catch (Exception e)
{
Logger.ErrorFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, e.FormatException());
}
}
}