private TopicStatisticsRecord ProcessTopic(string consumerGroup, string topic)
{
var topicPartitionMap = ZkUtils.GetPartitionsForTopics(this.zkClient, new[] { topic });
var topicState = new TopicStatisticsRecord
{
Topic = topic,
PartitionsStat = new Dictionary <int, PartitionStatisticsRecord>(topicPartitionMap[topic].Count)
};
foreach (var partitionId in topicPartitionMap[topic])
{
try
{
var partitionIdInt = int.Parse(partitionId);
topicState.PartitionsStat[partitionIdInt] = this.ProcessPartition(consumerGroup, topic, partitionIdInt);
}
catch (NoLeaderForPartitionException exc)
{
Logger.ErrorFormat("Could not found a leader for partition {0}. Details: {1}", exc.PartitionId, exc.FormatException());
}
catch (BrokerNotAvailableException exc)
{
Logger.ErrorFormat("Could not found a broker information for broker {0} while processing partition {1}. Details: {2}", exc.BrokerId, partitionId, exc.FormatException());
}
catch (OffsetIsUnknowException exc)
{
Logger.ErrorFormat("Could not retrieve offset from broker {0} for topic {1} partition {2}. Details: {3}", exc.BrokerId, exc.Topic, exc.PartitionId, exc.FormatException());
}
}
return(topicState);
}
private PartitionStatisticsRecord ProcessPartition(string consumerGroup, string topic, int partitionId)
{
Logger.DebugFormat("Collecting consumer offset statistics for consumer group {0}, topic {1}, partition {2}",
consumerGroup, topic, partitionId);
// find partition leader and create a consumer instance
var leader = ZkUtils.GetLeaderForPartition(zkClient, topic, partitionId);
if (!leader.HasValue)
{
throw new NoLeaderForPartitionException(partitionId);
}
var consumer = GetConsumer(leader.Value);
// get current offset
long?currentOffset;
var partitionIdStr = partitionId.ToString(CultureInfo.InvariantCulture);
var znode = ZkUtils.GetConsumerPartitionOffsetPath(consumerGroup, topic, partitionIdStr);
var currentOffsetString = zkClient.ReadData <string>(znode, true);
if (currentOffsetString == null)
{
// if offset is not stored in ZooKeeper retrieve first offset from actual Broker
currentOffset =
ConsumerUtils.EarliestOrLatestOffset(consumer, topic, partitionId, OffsetRequest.EarliestTime);
if (!currentOffset.HasValue)
{
throw new OffsetIsUnknowException(topic, leader.Value, partitionId);
}
}
else
{
currentOffset = long.Parse(currentOffsetString);
}
// get last offset
var lastOffset =
ConsumerUtils.EarliestOrLatestOffset(consumer, topic, partitionId, OffsetRequest.LatestTime);
if (!lastOffset.HasValue)
{
throw new OffsetIsUnknowException(topic, leader.Value, partitionId);
}
var owner = zkClient.ReadData <string>(
ZkUtils.GetConsumerPartitionOwnerPath(consumerGroup, topic, partitionIdStr), true);
return(new PartitionStatisticsRecord
{
PartitionId = partitionId,
CurrentOffset = currentOffset.Value,
LastOffset = lastOffset.Value,
OwnerConsumerId = owner
});
}
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.CreateEphemeral(path, data);
}
catch (KeeperException.NoNodeException)
{
ZkUtils.CreateParentPath(zkClient, path);
zkClient.CreateEphemeral(path, data);
}
}
private Dictionary <int, TopicPartitionState> ProcessTopic(string topic)
{
// might throw NoPartitionsForTopicException
var topicPartitionMap = ZkUtils.GetPartitionsForTopics(this.zkClient, new[] { topic });
var partitionsMetadata = new Dictionary <int, TopicPartitionState>();
foreach (var partitionId in topicPartitionMap[topic])
{
var partitionIdInt = int.Parse(partitionId);
partitionsMetadata[partitionIdInt] = ZkUtils.GetPartitionState(this.zkClient, topic, partitionIdInt);
}
return(partitionsMetadata);
}
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.CreateEphemeral(path, data);
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
ZkUtils.CreateParentPath(zkClient, path);
zkClient.CreateEphemeral(path, data);
}
else
{
throw;
}
}
}
public void HandleDataChange(string dataPath, object data)
{
var dataPathSplited = dataPath.Split('/');
var t = dataPathSplited[dataPathSplited.Length - 4];
var p = int.Parse(dataPathSplited[dataPathSplited.Length - 2]);
this.leaderLock.Lock();
try
{
if (t == this.topic && p == this.partition)
{
if (this.oldLeaderOpt.HasValue == false)
{
Logger.DebugFormat(
"In leader existence listener on partition [{0}, {1}], leader has been created",
topic,
partition);
this.leaderExistsOrChanged.Signal();
}
else
{
var newLeaderOpt = ZkUtils.GetLeaderForPartition(this.zkClient, t, p);
if (newLeaderOpt.HasValue && newLeaderOpt.Value != this.oldLeaderOpt.Value)
{
Logger.DebugFormat("In leader change listener on partition [{0}, {1}], leader has been moved from {2} to {3}", topic, partition, oldLeaderOpt.Value, newLeaderOpt.Value);
this.leaderExistsOrChanged.Signal();
}
}
}
}
finally
{
this.leaderLock.Unlock();
}
}
public static IEnumerable <Broker> GetAllBrokersInCluster(IZooKeeperClient zkClient)
{
var brokerIds = zkClient.GetChildren(ZooKeeperClient.DefaultBrokerIdsPath).OrderBy(x => x).ToList();
return(ZkUtils.GetBrokerInfoFromIds(zkClient, brokerIds.Select(x => int.Parse(x))));
}
