private long ResetConsumerOffsets(string topic, int partitionId)
{
long offset;
switch (_config.AutoOffsetReset)
{
case OffsetRequest.SmallestTime:
offset = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTime:
offset = OffsetRequest.LatestTime;
break;
default:
return(0);
}
var requestInfo = new Dictionary <string, List <PartitionOffsetRequestInfo> >();
requestInfo[topic] = new List <PartitionOffsetRequestInfo> {
new PartitionOffsetRequestInfo(partitionId, offset, 1)
};
var request = new OffsetRequest(requestInfo);
OffsetResponse offsets = _simpleConsumer.GetOffsetsBefore(request);
var topicDirs = new ZKGroupTopicDirs(_config.GroupId, topic);
long offsetFound = offsets.ResponseMap[topic].First().Offsets[0];
Logger.InfoFormat("updating partition {0} with {1} offset {2}", partitionId, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsetFound);
ZkUtils.UpdatePersistentPath(_zkClient, topicDirs.ConsumerOffsetDir + "/" + partitionId, offsetFound.ToString(CultureInfo.InvariantCulture));
return(offsetFound);
}
private void DeletePartitionOwnershipFromZK(string topic, int partition)
{
var topicDirs = new ZKGroupTopicDirs(group, topic);
var znode = topicDirs.ConsumerOwnerDir + "/" + partition;
ZkUtils.DeletePath(this.parent.zkClient, znode);
Logger.DebugFormat("Consumer {0} releasing {1}", consumerIdString, znode);
}
/// <summary>
/// Commit offset of specified topic/partition.
/// Only used when customer has strong requirement for reprocess messages as few as possible.
/// </summary>
/// <param name="topic"></param>
/// <param name="partition"></param>
/// <param name="offset"></param>
/// <param name="setPosition">Indicates whether to set the fetcher's offset to the value committed. Default = true.</param>
public void CommitOffset(string topic, int partition, long offset, bool setPosition = true)
{
this.EnsuresNotDisposed();
if (this.GetZkClient() == null)
{
return;
}
if (this.config.AutoCommit == true)
{
throw new ArgumentException(string.Format("When do commit offset with desired partition and offset, must set AutoCommit of ConsumerConfiguration as false!"));
}
try
{
IDictionary <int, PartitionTopicInfo> topicPartitionInfo = topicRegistry[topic];
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic);
PartitionTopicInfo partitionTopicInfo = topicPartitionInfo[partition];
if (partitionTopicInfo.ConsumeOffsetValid)
{
//Commit offset unconditionally. This would give consumes to decide which offset to read/skip
//if (offset > partitionTopicInfo.CommitedOffset)
try
{
ZkUtils.UpdatePersistentPath(GetZkClient(),
topicDirs.ConsumerOffsetDir + "/" +
partitionTopicInfo.PartitionId, offset.ToString());
partitionTopicInfo.CommitedOffset = offset;
if (setPosition)
{
partitionTopicInfo.ConsumeOffset = offset;
partitionTopicInfo.FetchOffset = offset;
}
}
catch (Exception ex)
{
Logger.ErrorFormat("error in CommitOffsets UpdatePersistentPath : {0}", ex.FormatException());
}
}
else
{
Logger.InfoFormat(
"Skip committing offset {0} for topic {1} because it is invalid (ZK session is disconnected)",
offset, partitionTopicInfo);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Commited offset {0} for topic {1}", offset, partitionTopicInfo);
}
}
catch (Exception ex)
{
Logger.ErrorFormat("exception during CommitOffsets: Topic:{0} Partition:{1} offset:{2} Exception:{3} ", topic, partition, offset, ex.FormatException());
}
}
/// <summary>
/// Produce the given number of messages, create a consumer with the given offset policy,
/// then reset the offset to the given value and consume until we get no new messages.
/// </summary>
/// <param name="numMessages"></param>
/// <param name="resetTo"></param>
/// <param name="offset"></param>
/// <returns>The count of messages received.</returns>
public int ResetAndConsume(int numMessages, string resetTo, long offset)
{
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 1000);
var producer = TestUtils.CreateProducer(
TestUtils.GetBrokerListFromConfigs(Configs), new DefaultEncoder(), new StringEncoder());
for (var i = 0; i < numMessages; i++)
{
producer.Send(new KeyedMessage <string, byte[]>(Topic, Topic, Encoding.UTF8.GetBytes("test")));
}
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000);
// update offset in zookeeper for consumer to jump "forward" in time
var dirs = new ZKGroupTopicDirs(Group, Topic);
var consumerConfig = TestUtils.CreateConsumerProperties(ZkConnect, Group, TestConsumer);
consumerConfig.AutoOffsetReset = resetTo;
consumerConfig.ConsumerTimeoutMs = 2000;
consumerConfig.FetchWaitMaxMs = 0;
TestUtils.UpdateConsumerOffset(consumerConfig, dirs.ConsumerOffsetDir + "/" + "0", offset);
Logger.InfoFormat("Update consumer offset to {0}", offset);
var consumerConnector = Consumer.Create(consumerConfig);
var messagesStream = consumerConnector.CreateMessageStreams(new Dictionary <string, int> {
{ Topic, 1 }
})[Topic].First();
var received = 0;
var iter = messagesStream.GetEnumerator();
try
{
for (var i = 0; i < numMessages; i++)
{
iter.MoveNext(); // will throw a timeout exception if the message isn't there
received++;
}
}
catch (ConsumerTimeoutException)
{
Logger.InfoFormat("consumer timeout out after receiving {0} messages", received);
}
finally
{
producer.Dispose();
consumerConnector.Shutdown();
}
return(received);
}
private void AddPartitionTopicInfo(ZKGroupTopicDirs topicDirs,
string partition,
string topic,
string consumerThreadId)
{
var partitionId = int.Parse(partition);
var partTopicInfoMap = topicRegistry[topic];
//find the leader for this partition
var leaderOpt = ZkUtils.GetLeaderForPartition(zkClient, topic, partitionId);
if (!leaderOpt.HasValue)
{
throw new NoBrokersForPartitionException(
string.Format("No leader available for partitions {0} on topic {1}", partition, topic));
}
Logger.InfoFormat("Leader for partition {0} for topic {1} is {2}", partition, topic, leaderOpt.Value);
var leader = leaderOpt.Value;
var znode = topicDirs.ConsumerOffsetDir + "/" + partition;
var offsetCommitedString = zkClient.ReadData <string>(znode, true);
//if first time starting a consumer, set the initial offset based on the config
long offset = -1;
long offsetCommited = -1;
if (offsetCommitedString != null)
{
offsetCommited = long.Parse(offsetCommitedString);
offset = offsetCommited + 1;
}
Logger.InfoFormat("Final offset {0} for topic {1} partition {2} OffsetCommited {3}"
, offset, topic, partition, offsetCommited);
var queue = queues[new Tuple <string, string>(topic, consumerThreadId)];
var partTopicInfo = new PartitionTopicInfo(
topic,
leader,
partitionId,
queue,
offsetCommited,
offset,
config.FetchSize,
offsetCommited);
partTopicInfoMap[partitionId] = partTopicInfo;
Logger.InfoFormat("{0} selected new offset {1}", partTopicInfo, offset);
}
private void ReleasePartitionOwnership()
{
foreach (KeyValuePair <string, IDictionary <Partition, PartitionTopicInfo> > item in topicRegistry)
{
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, item.Key);
foreach (var partition in item.Value.Keys)
{
string znode = topicDirs.ConsumerOwnerDir + "/" + partition.Name;
ZkUtils.DeletePath(zkClient, znode);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(CultureInfo.CurrentCulture, "Consumer {0} releasing {1}", this.consumerIdString, znode);
}
}
}
}
private bool ReflectPartitionOwnershipDecision(string topic, string partition, string consumerThreadId)
{
var ownershipReflected = false;
var topicDirs = new ZKGroupTopicDirs(config.GroupId, topic);
var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition;
try
{
Logger.InfoFormat("Consumer {0} will own partition {1} for topic {2}, will create ZK path {3}",
consumerThreadId, partition, topic, partitionOwnerPath);
try
{
zkClient.SlimLock.EnterWriteLock();
ZkUtils.CreateEphemeralPathExpectConflict(zkClient, partitionOwnerPath, consumerThreadId);
Logger.InfoFormat(
"Consumer {0} SUCC owned partition {1} for topic {2} . finish create ZK path {3} .",
consumerThreadId, partition, topic, partitionOwnerPath);
ownershipReflected = true;
}
catch (Exception ex)
{
Logger.InfoFormat(
"Consumer {0} FAILED owned partition {1} for topic {2} . finish create ZK path {3} error:{4}.",
consumerThreadId, partition, topic, partitionOwnerPath, ex.FormatException());
}
finally
{
zkClient.SlimLock.ExitWriteLock();
}
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NODEEXISTS)
{
Logger.InfoFormat(
"{0} failed to own partition {1} for topic {2}. Waiting for the partition owner to be deleted",
consumerThreadId, partition, topic);
ownershipReflected = false;
}
else
{
throw;
}
}
return(ownershipReflected);
}
private bool ProcessPartition(ZKGroupTopicDirs topicDirs, string partition, string topic, string consumerThreadId)
{
var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition;
try
{
ZkUtils.CreateEphemeralPathExpectConflict(zkClient, partitionOwnerPath, consumerThreadId);
}
catch (KeeperException.NodeExistsException)
{
//// The node hasn't been deleted by the original owner. So wait a bit and retry.
Logger.InfoFormat(CultureInfo.CurrentCulture, "waiting for the partition ownership to be deleted: {0}", partition);
return(false);
}
AddPartitionTopicInfo(topicDirs, partition, topic, consumerThreadId);
return(true);
}
private bool ProcessPartition(ZKGroupTopicDirs topicDirs,
string partition,
string topic,
string consumerThreadId,
List <string> curConsumers,
List <string> curPartitions,
CancellationTokenSource cancellationTokenSource)
{
var partitionOwned = false;
// Loop until we can successfully acquire partition, or are canceled by a
// future rebalance event
var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition;
while (true)
{
var currentPartitionOwner = zkClient.ReadData <string>(partitionOwnerPath, true);
if (currentPartitionOwner == null || currentPartitionOwner.Equals(consumerThreadId))
{
Logger.InfoFormat("{0} is null or equals our consumer ID. Reflect partition ownership in ZK",
partitionOwnerPath);
AddPartitionTopicInfo(topicDirs, partition, topic, consumerThreadId);
if (ReflectPartitionOwnershipDecision(topic, partition, consumerThreadId))
{
partitionOwned = true;
break;
}
}
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.Info(
"Rebalance operation has been canceled externally by a future rebalance event. Exiting immediately");
break;
}
Logger.InfoFormat("{0} exists with value {1}. Sleep and retry ownership attempt", partitionOwnerPath,
currentPartitionOwner);
Thread.Sleep(1000);
}
return(partitionOwned);
}
private bool ReflectPartitionOwnership(Dictionary <Tuple <string, string>, string> partitionOwnershipDecision)
{
var successfullyOwnedPartitions = new List <Tuple <string, string> >();
var partitionOwnershipSuccessful = new List <bool>();
foreach (var partitionOwner in partitionOwnershipDecision)
{
var topic = partitionOwner.Key.Item1;
var partition = partitionOwner.Key.Item2;
var consumerThreadId = partitionOwner.Value;
var topicDirs = new ZKGroupTopicDirs(config.GroupId, topic);
var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition;
try
{
ZkUtils.CreateEphemeralPathExpectConflict(zkClient, partitionOwnerPath, consumerThreadId);
Logger.InfoFormat("{0} successfully owned partition {1} for topic {2}", consumerThreadId, partition, topic);
successfullyOwnedPartitions.Add(new Tuple <string, string>(topic, partition));
partitionOwnershipSuccessful.Add(true);
}
catch (KeeperException.NodeExistsException)
{
Logger.InfoFormat("waiting for the partition owner to be deleted: {0}", partition);
partitionOwnershipSuccessful.Add(false);
}
}
var hasPartitionOwnershipFailed = partitionOwnershipSuccessful.Contains(false);
if (hasPartitionOwnershipFailed)
{
foreach (var topicAndPartition in successfullyOwnedPartitions)
{
var topicDirs = new ZKGroupTopicDirs(config.GroupId, topicAndPartition.Item1);
var znode = topicDirs.ConsumerOwnerDir + "/" + topicAndPartition.Item2;
ZkUtils.DeletePath(zkClient, znode);
Logger.DebugFormat("Consumer {0} releasing {1}", consumerIdString, znode);
}
return(false);
}
return(true);
}
/// <summary>
/// Commits the offsets of all messages consumed so far.
/// </summary>
public void CommitOffsets()
{
this.EnsuresNotDisposed();
if (this.zkClient == null)
{
return;
}
this.zkClient.SlimLock.EnterReadLock();
try
{
foreach (KeyValuePair <string, IDictionary <Partition, PartitionTopicInfo> > topic in topicRegistry)
{
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic.Key);
foreach (KeyValuePair <Partition, PartitionTopicInfo> partition in topic.Value)
{
var newOffset = partition.Value.GetConsumeOffset();
try
{
ZkUtils.UpdatePersistentPath(zkClient,
topicDirs.ConsumerOffsetDir + "/" +
partition.Value.Partition.Name, newOffset.ToString());
}
catch (Exception ex)
{
Logger.WarnFormat(CultureInfo.CurrentCulture, "exception during CommitOffsets: {0}", ex);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(CultureInfo.CurrentCulture, "Commited offset {0} for topic {1}",
newOffset, partition);
}
}
}
}
finally
{
this.zkClient.SlimLock.ExitReadLock();
}
}
private void AddPartitionTopicInfo(ZKGroupTopicDirs topicDirs, string partitionString, string topic, string consumerThreadId)
{
var partition = Partition.ParseFrom(partitionString);
var partTopicInfoMap = this.topicRegistry[topic];
var znode = topicDirs.ConsumerOffsetDir + "/" + partition.Name;
var offsetString = this.zkClient.ReadData <string>(znode, true);
long offset = string.IsNullOrEmpty(offsetString) ? 0 : long.Parse(offsetString, CultureInfo.InvariantCulture);
var queue = this.queues[new Tuple <string, string>(topic, consumerThreadId)];
var partTopicInfo = new PartitionTopicInfo(
topic,
partition.BrokerId,
partition,
queue,
offset,
offset,
this.config.FetchSize);
partTopicInfoMap.Add(partition, partTopicInfo);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(CultureInfo.CurrentCulture, "{0} selected new offset {1}", partTopicInfo, offset);
}
}
private long ResetConsumerOffsets(string topic, int partitionId)
{
long offset;
switch (_config.AutoOffsetReset)
{
case OffsetRequest.SmallestTime:
offset = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTime:
offset = OffsetRequest.LatestTime;
break;
default:
return 0;
}
var requestInfo = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
requestInfo[topic] = new List<PartitionOffsetRequestInfo> { new PartitionOffsetRequestInfo(partitionId, offset, 1) };
var request = new OffsetRequest(requestInfo);
OffsetResponse offsets = _simpleConsumer.GetOffsetsBefore(request);
var topicDirs = new ZKGroupTopicDirs(_config.GroupId, topic);
long offsetFound = offsets.ResponseMap[topic].First().Offsets[0];
Logger.InfoFormat("updating partition {0} with {1} offset {2}", partitionId, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsetFound);
ZkUtils.UpdatePersistentPath(_zkClient, topicDirs.ConsumerOffsetDir + "/" + partitionId, offsetFound.ToString(CultureInfo.InvariantCulture));
return offsetFound;
}
private bool Rebalance()
{
var myTopicThresdIdsMap = this.GetTopicCount(this.consumerIdString).GetConsumerThreadIdsPerTopic();
var cluster = new Cluster(zkClient);
var consumersPerTopicMap = this.GetConsumersPerTopic(this.config.GroupId);
var partitionsPerTopicMap = ZkUtils.GetPartitionsForTopics(this.zkClient, myTopicThresdIdsMap.Keys);
var relevantTopicThreadIdsMap = GetRelevantTopicMap(
myTopicThresdIdsMap,
partitionsPerTopicMap,
this.oldPartitionsPerTopicMap,
consumersPerTopicMap,
this.oldConsumersPerTopicMap);
if (relevantTopicThreadIdsMap.Count <= 0)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "Consumer {0} with {1} doesn't need to rebalance.", this.consumerIdString, consumersPerTopicMap);
return(true);
}
this.CloseFetchers(cluster, myTopicThresdIdsMap, this.zkConsumerConnector);
Logger.Info("Releasing parittion ownership");
this.ReleasePartitionOwnership();
var currentTopicRegistry = new ConcurrentDictionary <string, IDictionary <Partition, PartitionTopicInfo> >();
var partitionOwnershipDecision = new Dictionary <Tuple <string, string>, string>();
foreach (var item in myTopicThresdIdsMap)
{
currentTopicRegistry.GetOrAdd(item.Key, new ConcurrentDictionary <Partition, PartitionTopicInfo>());
var topicDirs = new ZKGroupTopicDirs(config.GroupId, item.Key);
var curConsumers = consumersPerTopicMap[item.Key];
var curPartitions = new List <string>(partitionsPerTopicMap[item.Key]);
var numberOfPartsPerConsumer = curPartitions.Count / curConsumers.Count;
var numberOfConsumersWithExtraPart = curPartitions.Count % curConsumers.Count;
Logger.InfoFormat(
CultureInfo.CurrentCulture,
"Consumer {0} rebalancing the following partitions: {1} for topic {2} with consumers: {3}",
this.consumerIdString,
string.Join(",", curPartitions),
item.Key,
string.Join(",", curConsumers));
foreach (string consumerThreadId in item.Value)
{
var myConsumerPosition = curConsumers.IndexOf(consumerThreadId);
if (myConsumerPosition < 0)
{
continue;
}
var startPart = (numberOfPartsPerConsumer * myConsumerPosition) +
Math.Min(myConsumerPosition, numberOfConsumersWithExtraPart);
var numberOfParts = numberOfPartsPerConsumer + (myConsumerPosition + 1 > numberOfConsumersWithExtraPart ? 0 : 1);
if (numberOfParts <= 0)
{
Logger.WarnFormat(CultureInfo.CurrentCulture, "No broker partitions consumed by consumer thread {0} for topic {1}", consumerThreadId, item.Key);
}
else
{
for (int i = startPart; i < startPart + numberOfParts; i++)
{
var partition = curPartitions[i];
Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} attempting to claim partition {1}", consumerThreadId, partition);
AddPartitionTopicInfo(currentTopicRegistry, topicDirs, partition, item.Key, consumerThreadId);
partitionOwnershipDecision.Add(new Tuple <string, string>(item.Key, partition), consumerThreadId);
}
}
}
}
if (ReflectPartitionOwnership(partitionOwnershipDecision))
{
zkClient.SlimLock.EnterWriteLock();
try
{
this.topicRegistry.Clear();
foreach (var item in currentTopicRegistry)
{
this.topicRegistry.Add(item);
}
this.UpdateFetcher(cluster);
}
finally
{
zkClient.SlimLock.ExitWriteLock();
}
this.oldPartitionsPerTopicMap = partitionsPerTopicMap;
this.oldConsumersPerTopicMap = consumersPerTopicMap;
return(true);
}
else
{
return(false);
}
}
private bool Rebalance(Cluster cluster)
{
var myTopicThreadIdsMap =
TopicCount.ConstructTopicCount(group, consumerIdString, parent.zkClient)
.GetConsumerThreadIdsPerTopic();
var consumersPerTopicMap = ZkUtils.GetConsumersPerTopic(parent.zkClient, group);
var brokers = ZkUtils.GetAllBrokersInCluster(parent.zkClient);
if (brokers.Count == 0)
{
// This can happen in a rare case when there are no brokers available in the cluster when the consumer is started.
// We log an warning and register for child changes on brokers/id so that rebalance can be triggered when the brokers
// are up.
Logger.Warn("no brokers found when trying to rebalance.");
parent.zkClient.SubscribeChildChanges(ZkUtils.BrokerIdsPath, parent.loadBalancerListener);
return(true);
}
else
{
var partitionsAssignmentPerTopicMap = ZkUtils.GetPartitionAssignmentForTopics(
parent.zkClient, myTopicThreadIdsMap.Keys.ToList());
var partitionsPerTopicMap = partitionsAssignmentPerTopicMap.ToDictionary(
p => p.Key, p => p.Value.Keys.OrderBy(x => x).ToList());
/**
* fetchers must be stopped to avoid Data duplication, since if the current
* rebalancing attempt fails, the partitions that are released could be owned by another consumer.
* But if we don't stop the fetchers first, this consumer would continue returning Data for released
* partitions in parallel. So, not stopping the fetchers leads to duplicate Data.
*/
this.CloseFetchers(cluster, (IDictionary <string, IList <KafkaStream <TKey, TValue> > >)KafkaMessageAndMetadataStreams, myTopicThreadIdsMap);
this.ReleasePartitionOwnership(parent.topicRegistry);
var partitionOwnershipDecision = new Dictionary <Tuple <string, int>, string>();
var currentTopicRegistry = new Pool <string, Pool <int, PartitionTopicInfo> >();
foreach (var topicAndConsumerThreadIsSet in myTopicThreadIdsMap)
{
var topic = topicAndConsumerThreadIsSet.Key;
var consumerThreadIdSet = topicAndConsumerThreadIsSet.Value;
currentTopicRegistry[topic] = new Pool <int, PartitionTopicInfo>();
var topicDirs = new ZKGroupTopicDirs(group, topic);
var curConsumers = consumersPerTopicMap.Get(topic);
var curPartitions = partitionsPerTopicMap.Get(topic);
var nPartsPerConsumer = curPartitions.Count / curConsumers.Count;
var nConsumersWithExtraPart = curPartitions.Count % curConsumers.Count;
Logger.InfoFormat("Consumer {0} rebalancing the following partitions: {1} for topic {2} with consumers: {3}", consumerIdString, string.Join(",", curPartitions), topic, string.Join(",", curConsumers));
foreach (var consumerThreadId in consumerThreadIdSet)
{
var myConsumerPosition = curConsumers.IndexOf(consumerThreadId);
Contract.Assert(myConsumerPosition >= 0);
var startPart = (nPartsPerConsumer * myConsumerPosition)
+ Math.Min(nConsumersWithExtraPart, myConsumerPosition);
var nParts = nPartsPerConsumer + (myConsumerPosition + 1 > nConsumersWithExtraPart ? 0 : 1);
/**
* Range-partition the sorted partitions to consumers for better locality.
* The first few consumers pick up an extra partition, if any.
*/
if (nParts <= 0)
{
Logger.WarnFormat(
"No broker partitions consumed by consumer thread {0} for topic {1}",
consumerThreadId,
topic);
}
else
{
for (var i = startPart; i < startPart + nParts; i++)
{
var partition = curPartitions[i];
Logger.InfoFormat("{0} attempting to claim partition {1}", consumerThreadId, partition);
this.AddPartitionTopicInfo(currentTopicRegistry, topicDirs, partition, topic, consumerThreadId);
// record the partition ownership decision
partitionOwnershipDecision[Tuple.Create(topic, partition)] = consumerThreadId;
}
}
}
}
/**
* move the partition ownership here, since that can be used to indicate a truly successful rebalancing attempt
* A rebalancing attempt is completed successfully only after the fetchers have been started correctly
*/
if (this.ReflectPartitionOwnershipDecision(partitionOwnershipDecision))
{
Logger.Info("Updating the cache");
Logger.Debug("Partitions per topic cache " + JObject.FromObject(partitionsPerTopicMap).ToString(Formatting.None));
Logger.Debug("Consumers per topic cache " + JObject.FromObject(consumersPerTopicMap).ToString(Formatting.None));
parent.topicRegistry = currentTopicRegistry;
this.UpdateFetcher(cluster);
return(true);
}
else
{
return(false);
}
}
}
/// <summary>
/// Commits the offsets of all messages consumed so far.
/// </summary>
public void CommitOffsets()
{
EnsuresNotDisposed();
if (GetZkClient() == null)
{
return;
}
try
{
foreach (var topic in topicRegistry)
{
var topicDirs = new ZKGroupTopicDirs(config.GroupId, topic.Key);
foreach (var partition in topic.Value)
{
var newOffset = partition.Value.ConsumeOffset;
try
{
if (partition.Value.ConsumeOffsetValid)
{
// Save offsets unconditionally. Kafka's latestOffset for a particular topic-partition can go backward
// if a follwer which is not fully caught up becomes a leader. We still need to save the conumed offsets even then.
//skip only if we are trying to commit the same offset
if (newOffset != partition.Value.CommitedOffset)
{
try
{
ZkUtils.UpdatePersistentPath(GetZkClient(),
topicDirs.ConsumerOffsetDir + "/" +
partition.Value.PartitionId, newOffset.ToString());
partition.Value.CommitedOffset = newOffset;
}
catch (Exception ex)
{
Logger.ErrorFormat("error in CommitOffsets UpdatePersistentPath : {0}",
ex.FormatException());
}
}
}
else
{
Logger.InfoFormat(
"Skip committing offset {0} for topic {1} because it is invalid (ZK session is disconnected)",
newOffset, partition);
}
}
catch (Exception ex)
{
Logger.WarnFormat("exception during CommitOffsets: {0}", ex.FormatException());
}
//if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Commited offset {0} for topic {1}", newOffset, partition);
}
}
}
}
catch (Exception ex)
{
Logger.ErrorFormat("error in CommitOffsets : {0}", ex.FormatException());
}
}
public ZookeeperConsumerConnectorTest()
{
this.dirs = new ZKGroupTopicDirs(Group, Topic);
}
private void AddPartitionTopicInfo(ZKGroupTopicDirs topicDirs, string partition, string topic, string consumerThreadId)
{
var partitionId = int.Parse(partition);
var partTopicInfoMap = this.topicRegistry[topic];
//find the leader for this partition
var leaderOpt = ZkUtils.GetLeaderForPartition(this.zkClient, topic, partitionId);
if (!leaderOpt.HasValue)
{
throw new NoBrokersForPartitionException(string.Format("No leader available for partitions {0} on topic {1}", partition, topic));
}
else
{
Logger.InfoFormat("Leader for partition {0} for topic {1} is {2}", partition, topic, leaderOpt.Value);
}
var leader = leaderOpt.Value;
var znode = topicDirs.ConsumerOffsetDir + "/" + partition;
var offsetString = this.zkClient.ReadData <string>(znode, true);
//if first time starting a consumer, set the initial offset based on the config
long offset = 0;
long offsetCommited = 0;
if (offsetString == null)
{
switch (config.AutoOffsetReset)
{
case OffsetRequest.SmallestTime:
offset = this.EarliestOrLatestOffset(topic, leader, partitionId, OffsetRequest.EarliestTime);
break;
case OffsetRequest.LargestTime:
offset = this.EarliestOrLatestOffset(topic, leader, partitionId, OffsetRequest.LatestTime);
break;
default:
throw new ConfigurationErrorsException("Wrong value in autoOffsetReset in ConsumerConfig");
}
}
else
{
offsetCommited = long.Parse(offsetString);
long latestOffset = this.EarliestOrLatestOffset(topic, leader, partitionId, OffsetRequest.LatestTime);
offset = Math.Min(offsetCommited + 1, latestOffset);
Logger.InfoFormat("Final offset {0} for topic {1} partition {2} OffsetCommited {3} latestOffset {4}"
, offset, topic, partition, offsetCommited, latestOffset);
}
var queue = this.queues[new Tuple <string, string>(topic, consumerThreadId)];
var partTopicInfo = new PartitionTopicInfo(
topic,
leader,
partitionId,
queue,
offsetCommited,
offset,
offset,
this.config.FetchSize,
offsetCommited);
partTopicInfoMap[partitionId] = partTopicInfo;
Logger.InfoFormat("{0} selected new offset {1}", partTopicInfo, offset);
}
private bool Rebalance()
{
var myTopicThresdIdsMap = this.GetTopicCount(this.consumerIdString).GetConsumerThreadIdsPerTopic();
var cluster = new Cluster(zkClient);
var consumersPerTopicMap = this.GetConsumersPerTopic(this.config.GroupId);
var partitionsPerTopicMap = ZkUtils.GetPartitionsForTopics(this.zkClient, myTopicThresdIdsMap.Keys);
var relevantTopicThreadIdsMap = GetRelevantTopicMap(
myTopicThresdIdsMap,
partitionsPerTopicMap,
this.oldPartitionsPerTopicMap,
consumersPerTopicMap,
this.oldConsumersPerTopicMap);
if (relevantTopicThreadIdsMap.Count <= 0)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "Consumer {0} with {1} doesn't need to rebalance.", this.consumerIdString, consumersPerTopicMap);
return(true);
}
Logger.Info("Committing all offsets");
this.zkConsumerConnector.CommitOffsets();
Logger.Info("Releasing parittion ownership");
this.ReleasePartitionOwnership();
var queuesToBeCleared = new List <BlockingCollection <FetchedDataChunk> >();
foreach (var item in relevantTopicThreadIdsMap)
{
this.topicRegistry.Remove(item.Key);
this.topicRegistry.Add(item.Key, new Dictionary <Partition, PartitionTopicInfo>());
var topicDirs = new ZKGroupTopicDirs(config.GroupId, item.Key);
var curConsumers = consumersPerTopicMap[item.Key];
var curPartitions = new List <string>(partitionsPerTopicMap[item.Key]);
var numberOfPartsPerConsumer = curPartitions.Count / curConsumers.Count;
var numberOfConsumersWithExtraPart = curPartitions.Count % curConsumers.Count;
Logger.InfoFormat(
CultureInfo.CurrentCulture,
"Consumer {0} rebalancing the following partitions: {1} for topic {2} with consumers: {3}",
this.consumerIdString,
string.Join(",", curPartitions),
item.Key,
string.Join(",", curConsumers));
foreach (string consumerThreadId in item.Value)
{
var myConsumerPosition = curConsumers.IndexOf(consumerThreadId);
if (myConsumerPosition < 0)
{
continue;
}
var startPart = (numberOfPartsPerConsumer * myConsumerPosition) +
Math.Min(myConsumerPosition, numberOfConsumersWithExtraPart);
var numberOfParts = numberOfPartsPerConsumer + (myConsumerPosition + 1 > numberOfConsumersWithExtraPart ? 0 : 1);
if (numberOfParts <= 0)
{
Logger.WarnFormat(CultureInfo.CurrentCulture, "No broker partitions consumed by consumer thread {0} for topic {1}", consumerThreadId, item.Key);
}
else
{
for (int i = startPart; i < startPart + numberOfParts; i++)
{
var partition = curPartitions[i];
Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} attempting to claim partition {1}", consumerThreadId, partition);
if (!this.ProcessPartition(topicDirs, partition, item.Key, consumerThreadId))
{
return(false);
}
}
queuesToBeCleared.Add(queues[new Tuple <string, string>(item.Key, consumerThreadId)]);
}
}
}
this.UpdateFetcher(cluster, queuesToBeCleared);
this.oldPartitionsPerTopicMap = partitionsPerTopicMap;
this.oldConsumersPerTopicMap = consumersPerTopicMap;
return(true);
}
private bool Rebalance(Cluster.Cluster cluster, CancellationTokenSource cancellationTokenSource)
{
var topicCount = GetTopicCount(consumerIdString);
var topicThreadIdsMap = topicCount.GetConsumerThreadIdsPerTopic();
if (!topicThreadIdsMap.Any())
{
Logger.ErrorFormat("Consumer ID is not registered to any topics in ZK. Exiting rebalance");
return(false);
}
var consumersPerTopicMap = GetConsumersPerTopic(config.GroupId);
var brokers = ZkUtils.GetAllBrokersInCluster(zkClient);
if (!brokers.Any())
{
Logger.Warn("No brokers found when trying to rebalance.");
zkClient.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, this);
zkConsumerConnector.subscribedChildCollection.Add(
new Tuple <string, IZooKeeperChildListener>(ZooKeeperClient.DefaultBrokerIdsPath, this));
Logger.ErrorFormat(
"Subscribe count: subscribedChildCollection:{0} , subscribedZookeeperStateCollection:{1} subscribedZookeeperDataCollection:{2} "
, zkConsumerConnector.subscribedChildCollection.Count,
zkConsumerConnector.subscribedZookeeperStateCollection.Count,
zkConsumerConnector.subscribedZookeeperDataCollection.Count);
return(false);
}
var partitionsPerTopicMap = ZkUtils.GetPartitionsForTopics(zkClient, topicThreadIdsMap.Keys);
// Check if we've been canceled externally before we dive into the rebalance
if (cancellationTokenSource.IsCancellationRequested)
{
Logger.ErrorFormat(
"Rebalance operation has been canceled externally by a future rebalance event. Exiting immediately");
return(false);
}
CloseFetchers(cluster, topicThreadIdsMap, zkConsumerConnector);
ReleasePartitionOwnership(topicThreadIdsMap);
try
{
foreach (var item in topicThreadIdsMap)
{
var topic = item.Key;
var consumerThreadIdSet = item.Value;
topicRegistry.Add(topic, new ConcurrentDictionary <int, PartitionTopicInfo>());
var topicDirs = new ZKGroupTopicDirs(config.GroupId, topic);
var curConsumers = new List <string>(consumersPerTopicMap[topic]);
curConsumers.Sort();
var curPartitions = partitionsPerTopicMap[topic].OrderBy(p => int.Parse(p)).ToList();
Logger.InfoFormat(
"{4} Partitions. {5} ConsumerClients. Consumer {0} rebalancing the following partitions: {1} for topic {2} with consumers: {3}",
consumerIdString,
string.Join(",", curPartitions),
topic,
string.Join(",", curConsumers),
curPartitions.Count,
curConsumers.Count);
var numberOfPartsPerConsumer = curPartitions.Count / curConsumers.Count;
Logger.Info("Number of partitions per consumer is: " + numberOfPartsPerConsumer);
var numberOfConsumersWithExtraPart = curPartitions.Count % curConsumers.Count;
Logger.Info("Number of consumers with an extra partition are: " + numberOfConsumersWithExtraPart);
foreach (var consumerThreadId in consumerThreadIdSet)
{
var myConsumerPosition = curConsumers.IndexOf(consumerThreadId);
Logger.Info("Consumer position for consumer " + consumerThreadId + " is: " +
myConsumerPosition);
if (myConsumerPosition < 0)
{
continue;
}
var startPart = numberOfPartsPerConsumer * myConsumerPosition +
Math.Min(myConsumerPosition, numberOfConsumersWithExtraPart);
Logger.Info("Starting partition is: " + startPart);
var numberOfParts = numberOfPartsPerConsumer +
(myConsumerPosition + 1 > numberOfConsumersWithExtraPart ? 0 : 1);
Logger.Info("Number of partitions to work on is: " + numberOfParts);
if (numberOfParts <= 0)
{
Logger.InfoFormat("No broker partitions consumed by consumer thread {0} for topic {1}",
consumerThreadId, item.Key);
}
else
{
for (var i = startPart; i < startPart + numberOfParts; i++)
{
var partition = curPartitions[i];
Logger.InfoFormat("{0} attempting to claim partition {1}", consumerThreadId, partition);
var ownPartition = ProcessPartition(topicDirs, partition, topic, consumerThreadId,
curConsumers, curPartitions, cancellationTokenSource);
if (!ownPartition)
{
Logger.InfoFormat(
"{0} failed to claim partition {1} for topic {2}. Exiting rebalance",
consumerThreadId, partition, topic);
return(false);
}
}
}
}
}
}
catch (Exception ex)
{
Logger.ErrorFormat("error when rebalance: {0}", ex.FormatException());
return(false);
}
// If we get here, we know that we have owned all partitions successfully,
// therefore it is safe to update fetcher threads and begin dequeuing
Logger.Info("All partitions were successfully owned. Updating fetchers");
UpdateFetcher(cluster);
return(true);
}
