private PartitionStatisticsRecord ProcessPartition(string consumerGroup, string topic, int partitionId) { Logger.DebugFormat("Collecting consumer offset statistics for consumer group {0}, topic {1}, partition {2}", consumerGroup, topic, partitionId); // find partition leader and create a consumer instance var leader = ZkUtils.GetLeaderForPartition(zkClient, topic, partitionId); if (!leader.HasValue) { throw new NoLeaderForPartitionException(partitionId); } var consumer = GetConsumer(leader.Value); // get current offset long?currentOffset; var partitionIdStr = partitionId.ToString(CultureInfo.InvariantCulture); var znode = ZkUtils.GetConsumerPartitionOffsetPath(consumerGroup, topic, partitionIdStr); var currentOffsetString = zkClient.ReadData <string>(znode, true); if (currentOffsetString == null) { // if offset is not stored in ZooKeeper retrieve first offset from actual Broker currentOffset = ConsumerUtils.EarliestOrLatestOffset(consumer, topic, partitionId, OffsetRequest.EarliestTime); if (!currentOffset.HasValue) { throw new OffsetIsUnknowException(topic, leader.Value, partitionId); } } else { currentOffset = long.Parse(currentOffsetString); } // get last offset var lastOffset = ConsumerUtils.EarliestOrLatestOffset(consumer, topic, partitionId, OffsetRequest.LatestTime); if (!lastOffset.HasValue) { throw new OffsetIsUnknowException(topic, leader.Value, partitionId); } var owner = zkClient.ReadData <string>( ZkUtils.GetConsumerPartitionOwnerPath(consumerGroup, topic, partitionIdStr), true); return(new PartitionStatisticsRecord { PartitionId = partitionId, CurrentOffset = currentOffset.Value, LastOffset = lastOffset.Value, OwnerConsumerId = owner }); }
public void HandleDataChange(string dataPath, object data) { var dataPathSplited = dataPath.Split('/'); var t = dataPathSplited[dataPathSplited.Length - 4]; var p = int.Parse(dataPathSplited[dataPathSplited.Length - 2]); this.leaderLock.Lock(); try { if (t == this.topic && p == this.partition) { if (this.oldLeaderOpt.HasValue == false) { Logger.DebugFormat( "In leader existence listener on partition [{0}, {1}], leader has been created", topic, partition); this.leaderExistsOrChanged.Signal(); } else { var newLeaderOpt = ZkUtils.GetLeaderForPartition(this.zkClient, t, p); if (newLeaderOpt.HasValue && newLeaderOpt.Value != this.oldLeaderOpt.Value) { Logger.DebugFormat("In leader change listener on partition [{0}, {1}], leader has been moved from {2} to {3}", topic, partition, oldLeaderOpt.Value, newLeaderOpt.Value); this.leaderExistsOrChanged.Signal(); } } } } finally { this.leaderLock.Unlock(); } }