/// <summary>
/// Commits the offsets of all messages consumed so far.
/// </summary>
public void CommitOffsets()
{
this.EnsuresNotDisposed();
if (this.zkClient == null)
{
return;
}
foreach (KeyValuePair <string, IDictionary <Partition, PartitionTopicInfo> > topic in topicRegistry)
{
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic.Key);
foreach (KeyValuePair <Partition, PartitionTopicInfo> partition in topic.Value)
{
var newOffset = partition.Value.GetConsumeOffset();
try
{
ZkUtils.UpdatePersistentPath(zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Value.Partition.Name, newOffset.ToString());
}
catch (Exception ex)
{
Logger.WarnFormat(CultureInfo.CurrentCulture, "exception during CommitOffsets: {0}", ex);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(CultureInfo.CurrentCulture, "Commited offset {0} for topic {1}", newOffset, partition);
}
}
}
}
private long ResetConsumerOffsets(string topic, Partition partition)
{
long offset;
switch (this.config.AutoOffsetReset)
{
case OffsetRequest.SmallestTime:
offset = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTime:
offset = OffsetRequest.LatestTime;
break;
default:
return(-1);
}
var request = new OffsetRequest(topic, partition.PartId, offset, 1);
var offsets = this.simpleConsumer.GetOffsetsBefore(request);
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic);
Logger.InfoFormat(CultureInfo.CurrentCulture, "updating partition {0} with {1} offset {2}", partition.Name, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsets[0]);
ZkUtils.UpdatePersistentPath(this.zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Name, offsets[0].ToString());
return(offsets[0]);
}
public void CommitOffsets()
{
if (zkClient == null)
{
Logger.Error("zk client is null. Cannot commit offsets");
return;
}
foreach (var topicAndInfo in TopicRegistry)
{
var topic = topicAndInfo.Key;
var infos = topicAndInfo.Value;
var topicDirs = new ZKGroupTopicDirs(this.Config.GroupId, topic);
foreach (var info in infos.Values)
{
var newOffset = info.GetConsumeOffset();
if (newOffset != checkpointedOffsets[new TopicAndPartition(topic, info.PartitionId)])
{
try
{
ZkUtils.UpdatePersistentPath(
zkClient, topicDirs.ConsumerOffsetDir + "/" + info.PartitionId, newOffset.ToString(CultureInfo.InvariantCulture));
checkpointedOffsets[new TopicAndPartition(topic, info.PartitionId)] = newOffset;
}
catch (Exception e)
{
// log it and let it go
Logger.Warn("Exception during commitOffsets", e);
}
Logger.DebugFormat("Committed offset {0} for topic {1}", newOffset, info);
}
}
}
}
private void AddPartitionTopicInfo(
Pool <string, Pool <int, PartitionTopicInfo> > currentTopicRegistry,
ZKGroupTopicDirs topicDirs,
int partition,
string topic,
string consumerThreadId)
{
var partTopicInfoMap = currentTopicRegistry.Get(topic);
var znode = topicDirs.ConsumerOffsetDir + "/" + partition;
var offsetString = ZkUtils.ReadDataMaybeNull(parent.zkClient, znode).Item1;
// If first time starting a consumer, set the initial offset to -1
var offset = (offsetString != null) ? long.Parse(offsetString) : PartitionTopicInfo.InvalidOffset;
var queue = parent.topicThreadIdAndQueues.Get(Tuple.Create(topic, consumerThreadId));
var consumedOffset = new AtomicLong(offset);
var fetchedOffset = new AtomicLong(offset);
var partTopicInfo = new PartitionTopicInfo(
topic,
partition,
queue,
consumedOffset,
fetchedOffset,
new AtomicInteger(parent.Config.FetchMessageMaxBytes),
parent.Config.ClientId);
partTopicInfoMap[partition] = partTopicInfo;
Logger.DebugFormat("{0} selected new offset {1}", partTopicInfo, offset);
parent.checkpointedOffsets[new TopicAndPartition(topic, partition)] = offset;
}
private void ReleasePartitionOwnership()
{
zkClient.SlimLock.EnterWriteLock();
try
{
foreach (KeyValuePair <string, IDictionary <Partition, PartitionTopicInfo> > item in topicRegistry)
{
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, item.Key);
foreach (var partition in item.Value.Keys)
{
string znode = topicDirs.ConsumerOwnerDir + "/" + partition.Name;
ZkUtils.DeletePath(zkClient, znode);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat(CultureInfo.CurrentCulture, "Consumer {0} releasing {1}",
this.consumerIdString, znode);
}
}
topicRegistry.Remove(item.Key);
}
}
finally
{
zkClient.SlimLock.ExitWriteLock();
}
}
private long ResetConsumerOffsets(string topic, int partitionId)
{
long offset;
switch (_config.AutoOffsetReset)
{
case OffsetRequest.SmallestTime:
offset = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTime:
offset = OffsetRequest.LatestTime;
break;
default:
return(0);
}
var requestInfo = new Dictionary <string, List <PartitionOffsetRequestInfo> >();
requestInfo[topic] = new List <PartitionOffsetRequestInfo> {
new PartitionOffsetRequestInfo(partitionId, offset, 1)
};
var request = new OffsetRequest(requestInfo);
OffsetResponse offsets = _simpleConsumer.GetOffsetsBefore(request);
var topicDirs = new ZKGroupTopicDirs(_config.GroupId, topic);
long offsetFound = offsets.ResponseMap[topic].First().Offsets[0];
Logger.InfoFormat("updating partition {0} with {1} offset {2}", partitionId, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsetFound);
ZkUtils.UpdatePersistentPath(_zkClient, topicDirs.ConsumerOffsetDir + "/" + partitionId, offsetFound.ToString(CultureInfo.InvariantCulture));
return(offsetFound);
}
internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount)
{
this.EnsuresNotDisposed();
Logger.InfoFormat(CultureInfo.CurrentCulture, "begin registering consumer {0} in ZK", consumerIdString);
ZkUtils.CreateEphemeralPathExpectConflict(this.zkClient, dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString());
Logger.InfoFormat(CultureInfo.CurrentCulture, "end registering consumer {0} in ZK", consumerIdString);
}
/// <summary>Pre-creating bookkeeper metadata path in zookeeper.</summary>
/// <exception cref="System.IO.IOException"/>
private void PrepareBookKeeperEnv()
{
// create bookie available path in zookeeper if it doesn't exists
string zkAvailablePath = conf.Get(BkjmZkLedgersAvailablePath, BkjmZkLedgersAvailablePathDefault
);
CountDownLatch zkPathLatch = new CountDownLatch(1);
AtomicBoolean success = new AtomicBoolean(false);
AsyncCallback.StringCallback callback = new _StringCallback_255(zkAvailablePath,
success, zkPathLatch);
ZkUtils.AsyncCreateFullPathOptimistic(zkc, zkAvailablePath, new byte[0], ZooDefs.Ids
.OpenAclUnsafe, CreateMode.Persistent, callback, null);
try
{
if (!zkPathLatch.Await(zkc.GetSessionTimeout(), TimeUnit.Milliseconds) || !success
.Get())
{
throw new IOException("Couldn't create bookie available path :" + zkAvailablePath
+ ", timed out " + zkc.GetSessionTimeout() + " millis");
}
}
catch (Exception e)
{
Sharpen.Thread.CurrentThread().Interrupt();
throw new IOException("Interrupted when creating the bookie available path : " +
zkAvailablePath, e);
}
}
public void HandleDataChange(Events.ZooKeeperDataChangedEventArgs args)
{
int parsedLeader;
string nodePath = args.Path;
string nodeData = args.Data;
Logger.Info("A partition leader or ISR list has been updated. Determining if rebalancing is necessary");
if (!ZkUtils.TryParsePartitionLeader(nodeData, out parsedLeader))
{
Logger.Error("Skipping rebalancing. Failed to parse partition leader for path: " + nodePath + " from ZK node");
return;
}
else
{
if (!_partitionLeaderMap.ContainsKey(nodePath) || _partitionLeaderMap[nodePath] != parsedLeader)
{
string currentLeader = _partitionLeaderMap.ContainsKey(nodePath) ? _partitionLeaderMap[nodePath].ToString() : "null";
Logger.Info("Performing rebalancing. Leader value for path: " + nodePath + " has changed from " + currentLeader + " to " + parsedLeader);
_partitionLeaderMap[nodePath] = parsedLeader;
_rebalancer.AsyncRebalance();
}
else
{
Logger.Info("Skipping rebalancing. Leader value for path: " + nodePath + " is " + parsedLeader.ToString() + " and has not changed");
}
}
}
/// <summary>
/// Commits the offsets of all messages consumed so far.
/// </summary>
public void CommitOffsets()
{
this.EnsuresNotDisposed();
if (this.GetZkClient() == null)
{
return;
}
try
{
foreach (KeyValuePair <string, IDictionary <int, PartitionTopicInfo> > topic in topicRegistry)
{
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic.Key);
foreach (KeyValuePair <int, PartitionTopicInfo> partition in topic.Value)
{
var newOffset = partition.Value.ConsumeOffset;
try
{
if (partition.Value.ConsumeOffsetValid)
{
// Save offsets unconditionally. Kafka's latestOffset for a particular topic-partition can go backward
// if a follwer which is not fully caught up becomes a leader. We still need to save the conumed offsets even then.
//skip only if we are trying to commit the same offset
if (newOffset != partition.Value.CommitedOffset)
{
try
{
ZkUtils.UpdatePersistentPath(GetZkClient(),
topicDirs.ConsumerOffsetDir + "/" +
partition.Value.PartitionId, newOffset.ToString());
partition.Value.CommitedOffset = newOffset;
}
catch (Exception ex)
{
Logger.ErrorFormat("error in CommitOffsets UpdatePersistentPath : {0}", ex.FormatException());
}
}
}
else
{
Logger.InfoFormat("Skip committing offset {0} for topic {1} because it is invalid (ZK session is disconnected)", newOffset, partition);
}
}
catch (Exception ex)
{
Logger.WarnFormat("exception during CommitOffsets: {0}", ex.FormatException());
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Commited offset {0} for topic {1}", newOffset, partition);
}
}
}
}
catch (Exception ex)
{
Logger.ErrorFormat("error in CommitOffsets : {0}", ex.FormatException());
}
}
public override IDictionary <string, ISet <string> > GetConsumerThreadIdsPerTopic()
{
var wildcardTopics = ZkUtils.GetChildrenParentMayNotExist(this.ZkClient, ZkUtils.BrokerTopicsPath)
.Where(x => TopicFilter.IsTopicAllowed(x))
.ToList();
return(this.MakeConsumerThreadIdsPerTopic(this.ConsumerIdString, wildcardTopics.ToDictionary(x => x, v => this.NumStreams)));
}
private void DeletePartitionOwnershipFromZK(string topic, int partition)
{
var topicDirs = new ZKGroupTopicDirs(group, topic);
var znode = topicDirs.ConsumerOwnerDir + "/" + partition;
ZkUtils.DeletePath(this.parent.zkClient, znode);
Logger.DebugFormat("Consumer {0} releasing {1}", consumerIdString, znode);
}
internal static void Run(ProduceSimpleHelperOption produceOptions)
{
failedMessageCount = 0;
successMessageCount = 0;
sentBatchCount = 0;
produceMessagePerPartition = new Dictionary <int, int>();
produceMessagePerPartitionExpect = new Dictionary <int, int>();
PrepareSentMessages(produceOptions);
kafkaSimpleManagerConfig = new KafkaSimpleManagerConfiguration()
{
Zookeeper = produceOptions.Zookeeper,
MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize,
PartitionerClass = produceOptions.PartitionerClass
};
kafkaSimpleManagerConfig.Verify();
producerConfigTemplate = new ProducerConfiguration(
new List <BrokerConfiguration>()
{
}) //The Brokers will be replaced inside of KafkaSimpleManager
{
ForceToPartition = -1,
PartitionerClass = kafkaSimpleManagerConfig.PartitionerClass,
TotalNumPartitions = 0,
RequiredAcks = produceOptions.RequiredAcks,
AckTimeout = produceOptions.AckTimeout,
SendTimeout = produceOptions.SendTimeout,
ReceiveTimeout = produceOptions.ReceiveTimeout,
CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()),
BufferSize = produceOptions.BufferSize,
SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition
MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize)
};
using (ZooKeeperClient zkClient = new ZooKeeperClient(produceOptions.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
Dictionary <int, int[]> topicDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, produceOptions.Topic);
// -2 by default or customized partitioner class. (you need specify PartitionerClass by -l)
if (produceOptions.PartitionId == -2)
{
if (string.IsNullOrEmpty(kafkaSimpleManagerConfig.PartitionerClass))
{
throw new ArgumentException("The partitioer class must not be empty if you want to send to partition by partitioner.");
}
//if (producerConfigTemplate.TotalNumPartitions <= 0)
// throw new ArgumentException(string.Format("Please also specify the TotalNumPartitions if you want to send to partition by partitioner."));
ProduceByPartitionerClass(produceOptions, topicDataInZookeeper.Count);
}
else
{
ProduceToRandomOrSpecificPartition(produceOptions);
}
}
}
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient)
{
var dirs = new ZKGroupDirs(group);
var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1;
string subscriptionPattern;
IDictionary <string, int> topMap;
try
{
var parsedJson = JObject.Parse(topicCountString);
if (parsedJson != null)
{
var pattern = parsedJson.Get("pattern");
if (pattern != null)
{
subscriptionPattern = pattern.Value <string>();
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
var topMapObject = (IEnumerable <KeyValuePair <string, JToken> >)parsedJson.Get("subscription");
if (topMapObject != null)
{
topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value <int>());
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
catch (Exception e)
{
Logger.Error("error parsing consumer json string " + topicCountString, e);
throw;
}
var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern);
var hasBlackList = BlackListPattern.Equals(subscriptionPattern);
if (topMap.Count == 0 || !(hasWhiteList || hasBlackList))
{
return(new StaticTopicCount(consumerId, topMap));
}
else
{
var regex = topMap.First().Key;
var numStreams = topMap.First().Value;
TopicFilter filter = hasWhiteList ? (TopicFilter) new Whitelist(regex) : new Blacklist(regex);
return(new WildcardTopicCount(zkClient, consumerId, filter, numStreams));
}
}
public static void UpdateConsumerOffset(ConsumerConfig config, string path, long offset)
{
var zkClient = new ZkClient(
config.ZooKeeper.ZkConnect,
config.ZooKeeper.ZkSessionTimeoutMs,
config.ZooKeeper.ZkConnectionTimeoutMs,
new ZkStringSerializer());
ZkUtils.UpdatePersistentPath(zkClient, path, offset.ToString(CultureInfo.InvariantCulture));
}
/// <summary>
/// Commit offset of specified topic/partition.
/// Only used when customer has strong requirement for reprocess messages as few as possible.
/// </summary>
/// <param name="topic"></param>
/// <param name="partition"></param>
/// <param name="offset"></param>
/// <param name="setPosition">Indicates whether to set the fetcher's offset to the value committed. Default = true.</param>
public void CommitOffset(string topic, int partition, long offset, bool setPosition = true)
{
this.EnsuresNotDisposed();
if (this.GetZkClient() == null)
{
return;
}
if (this.config.AutoCommit == true)
{
throw new ArgumentException(string.Format("When do commit offset with desired partition and offset, must set AutoCommit of ConsumerConfiguration as false!"));
}
try
{
IDictionary <int, PartitionTopicInfo> topicPartitionInfo = topicRegistry[topic];
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic);
PartitionTopicInfo partitionTopicInfo = topicPartitionInfo[partition];
if (partitionTopicInfo.ConsumeOffsetValid)
{
//Commit offset unconditionally. This would give consumes to decide which offset to read/skip
//if (offset > partitionTopicInfo.CommitedOffset)
try
{
ZkUtils.UpdatePersistentPath(GetZkClient(),
topicDirs.ConsumerOffsetDir + "/" +
partitionTopicInfo.PartitionId, offset.ToString());
partitionTopicInfo.CommitedOffset = offset;
if (setPosition)
{
partitionTopicInfo.ConsumeOffset = offset;
partitionTopicInfo.FetchOffset = offset;
}
}
catch (Exception ex)
{
Logger.ErrorFormat("error in CommitOffsets UpdatePersistentPath : {0}", ex.FormatException());
}
}
else
{
Logger.InfoFormat(
"Skip committing offset {0} for topic {1} because it is invalid (ZK session is disconnected)",
offset, partitionTopicInfo);
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Commited offset {0} for topic {1}", offset, partitionTopicInfo);
}
}
catch (Exception ex)
{
Logger.ErrorFormat("exception during CommitOffsets: Topic:{0} Partition:{1} offset:{2} Exception:{3} ", topic, partition, offset, ex.FormatException());
}
}
public static void CreateTopic(
ZkClient zkClient,
string topic,
int partitions,
int replicationFactor,
Dictionary <string, string> topicConfig)
{
var brokerList = ZkUtils.GetSortedBrokerList(zkClient);
var replicaAssigment = AssignReplicasToBrokers(brokerList, partitions, replicationFactor);
CreateOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, replicaAssigment, topicConfig);
}
public void StartWatchingTopicEvents()
{
var topicEventListener = new ZkTopicEventListener(this);
ZkUtils.MakeSurePersistentPathExists(zkClient, ZkUtils.BrokerTopicsPath);
this.zkClient.SubscribeStateChanges(new ZkSessionExpireListener(this, topicEventListener));
var topics = this.zkClient.SubscribeChildChanges(ZkUtils.BrokerTopicsPath, topicEventListener);
// call to bootstrap topic list
topicEventListener.HandleChildChange(ZkUtils.BrokerTopicsPath, topics);
}
public static int?WaitUntilLeaderIsElectedOrChanged(
ZkClient zkClient, string topic, int partition, long timeoutMs, int?oldLeaderOpt = null)
{
var leaderLock = new ReentrantLock();
var leaderExistsOrChanged = leaderLock.NewCondition();
if (oldLeaderOpt.HasValue == false)
{
Logger.InfoFormat("Waiting for leader to be elected for partition [{0},{1}]", topic, partition);
}
else
{
Logger.InfoFormat("Waiting for leader for partition [{0},{1}] to be changed from old leader {2}", topic, partition, oldLeaderOpt.Value);
}
leaderLock.Lock();
try
{
zkClient.SubscribeDataChanges(ZkUtils.GetTopicPartitionLeaderAndIsrPath(topic, partition), new LeaderExistsOrChangedListener(topic, partition, leaderLock, leaderExistsOrChanged, oldLeaderOpt, zkClient));
leaderExistsOrChanged.Await(TimeSpan.FromMilliseconds(timeoutMs));
// check if leader is elected
var leader = ZkUtils.GetLeaderForPartition(zkClient, topic, partition);
if (leader != null)
{
if (oldLeaderOpt.HasValue == false)
{
Logger.InfoFormat("Leader {0} is elected for partition [{1},{2}]", leader, topic, partition);
}
else
{
Logger.InfoFormat(
"Leader for partition [{0},{1}] is changed from {2} to {3}",
topic,
partition,
oldLeaderOpt.Value,
leader);
}
}
else
{
Logger.ErrorFormat("Timing out after {0} ms since leader is not elected for partition [{1},{2}]", timeoutMs, topic, partition);
}
return(leader);
}
finally
{
leaderLock.Unlock();
}
}
public void SyncedRebalance()
{
lock (parent.rebalanceLock)
{
if (parent.isShuttingDown.Get())
{
return;
}
else
{
for (var i = 1; i <= parent.Config.RebalanceMaxRetries; i++)
{
Logger.InfoFormat("begin rebalancing consumer {0} try #{1}", consumerIdString, i);
var done = false;
Cluster cluster = null;
try
{
cluster = ZkUtils.GetCluster(parent.zkClient);
done = this.Rebalance(cluster);
}
catch (Exception e)
{
/** occasionally, we may hit a ZK exception because the ZK state is changing while we are iterating.
* For example, a ZK node can disappear between the time we get all children and the time we try to get
* the value of a child. Just let this go since another rebalance will be triggered.
**/
Logger.Info("Exception during rebalance", e);
}
Logger.InfoFormat("end rebalancing consumer {0} try #{1}", consumerIdString, i);
if (done)
{
return;
}
else
{
/* Here the cache is at a risk of being stale. To take future rebalancing decisions correctly, we should
* clear the cache */
Logger.Info("Rebalancing attempt failed. Clearing the cache before the next rebalancing operation is triggered");
}
// stop all fetchers and clear all the queues to avoid Data duplication
CloseFetchersForQueues(cluster, (IDictionary <string, IList <KafkaStream <TKey, TValue> > >)KafkaMessageAndMetadataStreams, parent.topicThreadIdAndQueues.Select(x => x.Value).ToList());
Thread.Sleep(parent.Config.RebalanceBackoffMs);
}
}
}
throw new ConsumerRebalanceFailedException(
consumerIdString + " can't rebalance after " + parent.Config.RebalanceMaxRetries + " retries");
}
private void AddPartitionTopicInfo(ZKGroupTopicDirs topicDirs,
string partition,
string topic,
string consumerThreadId)
{
var partitionId = int.Parse(partition);
var partTopicInfoMap = topicRegistry[topic];
//find the leader for this partition
var leaderOpt = ZkUtils.GetLeaderForPartition(zkClient, topic, partitionId);
if (!leaderOpt.HasValue)
{
throw new NoBrokersForPartitionException(
string.Format("No leader available for partitions {0} on topic {1}", partition, topic));
}
Logger.InfoFormat("Leader for partition {0} for topic {1} is {2}", partition, topic, leaderOpt.Value);
var leader = leaderOpt.Value;
var znode = topicDirs.ConsumerOffsetDir + "/" + partition;
var offsetCommitedString = zkClient.ReadData <string>(znode, true);
//if first time starting a consumer, set the initial offset based on the config
long offset = -1;
long offsetCommited = -1;
if (offsetCommitedString != null)
{
offsetCommited = long.Parse(offsetCommitedString);
offset = offsetCommited + 1;
}
Logger.InfoFormat("Final offset {0} for topic {1} partition {2} OffsetCommited {3}"
, offset, topic, partition, offsetCommited);
var queue = queues[new Tuple <string, string>(topic, consumerThreadId)];
var partTopicInfo = new PartitionTopicInfo(
topic,
leader,
partitionId,
queue,
offsetCommited,
offset,
config.FetchSize,
offsetCommited);
partTopicInfoMap[partitionId] = partTopicInfo;
Logger.InfoFormat("{0} selected new offset {1}", partTopicInfo, offset);
}
private bool ReflectPartitionOwnershipDecision(
IDictionary <Tuple <string, int>, string> partitionOwnershipDecision)
{
var successfullyOwnedPartitions = new List <Tuple <string, int> >();
var partitionOwnershipSuccessful = partitionOwnershipDecision.Select(
partitionOwner =>
{
var topic = partitionOwner.Key.Item1;
var partition = partitionOwner.Key.Item2;
var consumerThreadId = partitionOwner.Value;
var partitionOwnerPath = ZkUtils.GetConsumerPartitionOwnerPath(group, topic, partition);
try
{
ZkUtils.CreateEphemeralPathExpectConflict(
parent.zkClient, partitionOwnerPath, consumerThreadId);
Logger.InfoFormat(
"{0} successfully owner partition {1} for topic {2}",
consumerThreadId,
partition,
topic);
successfullyOwnedPartitions.Add(Tuple.Create(topic, partition));
return(true);
}
catch (ZkNodeExistsException)
{
// The node hasn't been deleted by the original owner. So wait a bit and retry.
Logger.InfoFormat("waiting for the partition ownership to be deleted:" + partition);
return(false);
}
}).ToList();
var hasPartitionOwnershipFailed = partitionOwnershipSuccessful.Aggregate(
0, (sum, decision) => (sum + (decision ? 0 : 1)));
/* even if one of the partition ownership attempt has failed, return false */
if (hasPartitionOwnershipFailed > 0)
{
// remove all paths that we have owned in ZK
foreach (var topicAndPartition in successfullyOwnedPartitions)
{
this.DeletePartitionOwnershipFromZK(topicAndPartition.Item1, topicAndPartition.Item2);
}
return(false);
}
return(true);
}
internal WildcardStreamsHandler(
ZookeeperConsumerConnector parent,
TopicFilter topicFilter,
int numStreams,
IDecoder <TKey> keyDecoder,
IDecoder <TValue> valueDecoder)
{
this.parent = parent;
this.topicFilter = topicFilter;
this.numStreams = numStreams;
this.keyDecoder = keyDecoder;
this.valueDecoder = valueDecoder;
if (parent.messageStreamCreated.GetAndSet(true))
{
throw new Exception("Each consumer connector can create message streams by filter at most once.");
}
this.wildcardQueuesAndStreams = Enumerable.Range(1, numStreams).Select(e =>
{
var queue = new BlockingCollection <FetchedDataChunk>(this.parent.Config.QueuedMaxMessages);
var stream = new KafkaStream <TKey, TValue>(
queue,
this.parent.Config.ConsumerTimeoutMs,
keyDecoder,
valueDecoder,
this.parent.Config.ClientId);
return(Tuple.Create(queue, stream));
}).ToList();
this.wildcardTopics =
ZkUtils.GetChildrenParentMayNotExist(this.parent.zkClient, ZkUtils.BrokerTopicsPath)
.Where(topicFilter.IsTopicAllowed)
.ToList();
this.wildcardTopicCount = TopicCount.ConstructTopicCount(
this.parent.consumerIdString, topicFilter, numStreams, this.parent.zkClient);
this.dirs = new ZKGroupDirs(this.parent.Config.GroupId);
this.parent.RegisterConsumerInZK(dirs, this.parent.consumerIdString, this.wildcardTopicCount);
this.parent.ReinitializeConsumer(this.wildcardTopicCount, this.wildcardQueuesAndStreams);
// Topic events will trigger subsequent synced rebalances.
Logger.InfoFormat("Creating topic event watcher for topics {0}", topicFilter);
this.parent.wildcardTopicWatcher = new ZookeeperTopicEventWatcher(this.parent.zkClient, this);
}
private bool ReflectPartitionOwnershipDecision(string topic, string partition, string consumerThreadId)
{
var ownershipReflected = false;
var topicDirs = new ZKGroupTopicDirs(config.GroupId, topic);
var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition;
try
{
Logger.InfoFormat("Consumer {0} will own partition {1} for topic {2}, will create ZK path {3}",
consumerThreadId, partition, topic, partitionOwnerPath);
try
{
zkClient.SlimLock.EnterWriteLock();
ZkUtils.CreateEphemeralPathExpectConflict(zkClient, partitionOwnerPath, consumerThreadId);
Logger.InfoFormat(
"Consumer {0} SUCC owned partition {1} for topic {2} . finish create ZK path {3} .",
consumerThreadId, partition, topic, partitionOwnerPath);
ownershipReflected = true;
}
catch (Exception ex)
{
Logger.InfoFormat(
"Consumer {0} FAILED owned partition {1} for topic {2} . finish create ZK path {3} error:{4}.",
consumerThreadId, partition, topic, partitionOwnerPath, ex.FormatException());
}
finally
{
zkClient.SlimLock.ExitWriteLock();
}
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NODEEXISTS)
{
Logger.InfoFormat(
"{0} failed to own partition {1} for topic {2}. Waiting for the partition owner to be deleted",
consumerThreadId, partition, topic);
ownershipReflected = false;
}
else
{
throw;
}
}
return(ownershipReflected);
}
public void ReleaseAllPartitionOwnerships()
{
Logger.Info("Releasing all partition ownerships");
var consumerIdString = GetConsumerIdString();
foreach (var item in topicRegistry)
{
var topic = item.Key;
try
{
foreach (var partition in item.Value.Keys)
{
var partitionOwnerPath = ZkUtils.GetConsumerPartitionOwnerPath(config.GroupId, topic,
partition.ToString());
Logger.InfoFormat("Consumer {0} will delete ZK path {1} topic:{2} partition:{3} ",
consumerIdString, partitionOwnerPath, topic, partition);
try
{
GetZkClient().SlimLock.EnterWriteLock();
ZkUtils.DeletePath(GetZkClient(), partitionOwnerPath);
Logger.InfoFormat(
"Consumer {0} SUCC delete ZK path {1} topic:{2} partition:{3} succsessfully.",
consumerIdString, partitionOwnerPath, topic, partition);
}
catch (Exception ex)
{
Logger.ErrorFormat(
"Consumer {0} FAILED delete ZK path {1} topic:{2} partition:{3} error:{4}.",
consumerIdString, partitionOwnerPath, topic, partition, ex.FormatException());
}
finally
{
GetZkClient().SlimLock.ExitWriteLock();
}
}
}
catch (Exception ex)
{
Logger.ErrorFormat("error when call ZkUtils.DeletePath : {0}", ex.FormatException());
}
}
Logger.Info("Released all partition ownerships");
}
private bool ProcessPartition(ZKGroupTopicDirs topicDirs, string partition, string topic, string consumerThreadId)
{
var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition;
try
{
ZkUtils.CreateEphemeralPathExpectConflict(zkClient, partitionOwnerPath, consumerThreadId);
}
catch (KeeperException.NodeExistsException)
{
//// The node hasn't been deleted by the original owner. So wait a bit and retry.
Logger.InfoFormat(CultureInfo.CurrentCulture, "waiting for the partition ownership to be deleted: {0}", partition);
return(false);
}
AddPartitionTopicInfo(topicDirs, partition, topic, consumerThreadId);
return(true);
}
internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount)
{
this.EnsuresNotDisposed();
Logger.InfoFormat("begin registering consumer {0} in ZK", consumerIdString);
try
{
this.GetZkClient().SlimLock.EnterWriteLock();
ZkUtils.CreateEphemeralPathExpectConflict(this.GetZkClient(), dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString());
Logger.InfoFormat("successfully registering consumer {0} in ZK", consumerIdString);
}
catch (Exception ex)
{
Logger.ErrorFormat("error in RegisterConsumerInZk CreateEphemeralPathExpectConflict : {0}", ex.FormatException());
}
finally
{
GetZkClient().SlimLock.ExitWriteLock();
}
}
private static void WriteTopicPartitionAssignment(
ZkClient zkClient, string topic, Dictionary <int, List <int> > replicaAssignment, bool update)
{
var zkPath = ZkUtils.GetTopicPath(topic);
var jsonPartitonData =
ZkUtils.ReplicaAssignmentZkData(replicaAssignment.ToDictionary(x => x.Key.ToString(), v => v.Value));
if (!update)
{
Logger.InfoFormat("Topic creation: {0}", jsonPartitonData);
ZkUtils.CreatePersistentPath(zkClient, zkPath, jsonPartitonData);
}
else
{
Logger.InfoFormat("Topic update {0}" + jsonPartitonData);
ZkUtils.UpdatePersistentPath(zkClient, zkPath, jsonPartitonData);
}
Logger.DebugFormat("Updated path {0} with {1} for replica assignment", zkPath, jsonPartitonData);
}
public void AddProducers(ProducerConfiguration config)
{
var configuredBrokers = config.Brokers.Select(x => new Broker(x.BrokerId, x.Host, x.Port));
if (configuredBrokers.Any())
{
configuredBrokers.ForEach(this.AddProducer);
}
else if (this.zkClient != null)
{
Logger.DebugFormat("Connecting to {0} for creating sync producers for all brokers in the cluster",
config.ZooKeeper.ZkConnect);
var brokers = ZkUtils.GetAllBrokersInCluster(this.zkClient);
brokers.ForEach(this.AddProducer);
}
else
{
throw new IllegalStateException("No producers found from configuration and zk not setup.");
}
}
public static void CreateOrUpdateTopicPartitionAssignmentPathInZK(
ZkClient zkClient,
string topic,
Dictionary <int, List <int> > partitionReplicaAssignment,
Dictionary <string, string> config,
bool update = false)
{
// Note: validation was omited
var topicPath = ZkUtils.GetTopicPath(topic);
if (!update && zkClient.Exists(topicPath))
{
throw new KafkaException("Topic " + topic + " already exists.");
}
// write out the config if there is any, this isn't transactional with the partition assignments
WriteTopicConfig(zkClient, topic, config);
// create the partition assignment
WriteTopicPartitionAssignment(zkClient, topic, partitionReplicaAssignment, update);
}