/// <summary> /// Commits the offsets of all messages consumed so far. /// </summary> public void CommitOffsets() { this.EnsuresNotDisposed(); if (this.zkClient == null) { return; } foreach (KeyValuePair <string, IDictionary <Partition, PartitionTopicInfo> > topic in topicRegistry) { var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic.Key); foreach (KeyValuePair <Partition, PartitionTopicInfo> partition in topic.Value) { var newOffset = partition.Value.GetConsumeOffset(); try { ZkUtils.UpdatePersistentPath(zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Value.Partition.Name, newOffset.ToString()); } catch (Exception ex) { Logger.WarnFormat(CultureInfo.CurrentCulture, "exception during CommitOffsets: {0}", ex); } if (Logger.IsDebugEnabled) { Logger.DebugFormat(CultureInfo.CurrentCulture, "Commited offset {0} for topic {1}", newOffset, partition); } } } }
private long ResetConsumerOffsets(string topic, Partition partition) { long offset; switch (this.config.AutoOffsetReset) { case OffsetRequest.SmallestTime: offset = OffsetRequest.EarliestTime; break; case OffsetRequest.LargestTime: offset = OffsetRequest.LatestTime; break; default: return(-1); } var request = new OffsetRequest(topic, partition.PartId, offset, 1); var offsets = this.simpleConsumer.GetOffsetsBefore(request); var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic); Logger.InfoFormat(CultureInfo.CurrentCulture, "updating partition {0} with {1} offset {2}", partition.Name, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsets[0]); ZkUtils.UpdatePersistentPath(this.zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Name, offsets[0].ToString()); return(offsets[0]); }
public void CommitOffsets() { if (zkClient == null) { Logger.Error("zk client is null. Cannot commit offsets"); return; } foreach (var topicAndInfo in TopicRegistry) { var topic = topicAndInfo.Key; var infos = topicAndInfo.Value; var topicDirs = new ZKGroupTopicDirs(this.Config.GroupId, topic); foreach (var info in infos.Values) { var newOffset = info.GetConsumeOffset(); if (newOffset != checkpointedOffsets[new TopicAndPartition(topic, info.PartitionId)]) { try { ZkUtils.UpdatePersistentPath( zkClient, topicDirs.ConsumerOffsetDir + "/" + info.PartitionId, newOffset.ToString(CultureInfo.InvariantCulture)); checkpointedOffsets[new TopicAndPartition(topic, info.PartitionId)] = newOffset; } catch (Exception e) { // log it and let it go Logger.Warn("Exception during commitOffsets", e); } Logger.DebugFormat("Committed offset {0} for topic {1}", newOffset, info); } } } }
private void AddPartitionTopicInfo( Pool <string, Pool <int, PartitionTopicInfo> > currentTopicRegistry, ZKGroupTopicDirs topicDirs, int partition, string topic, string consumerThreadId) { var partTopicInfoMap = currentTopicRegistry.Get(topic); var znode = topicDirs.ConsumerOffsetDir + "/" + partition; var offsetString = ZkUtils.ReadDataMaybeNull(parent.zkClient, znode).Item1; // If first time starting a consumer, set the initial offset to -1 var offset = (offsetString != null) ? long.Parse(offsetString) : PartitionTopicInfo.InvalidOffset; var queue = parent.topicThreadIdAndQueues.Get(Tuple.Create(topic, consumerThreadId)); var consumedOffset = new AtomicLong(offset); var fetchedOffset = new AtomicLong(offset); var partTopicInfo = new PartitionTopicInfo( topic, partition, queue, consumedOffset, fetchedOffset, new AtomicInteger(parent.Config.FetchMessageMaxBytes), parent.Config.ClientId); partTopicInfoMap[partition] = partTopicInfo; Logger.DebugFormat("{0} selected new offset {1}", partTopicInfo, offset); parent.checkpointedOffsets[new TopicAndPartition(topic, partition)] = offset; }
private void ReleasePartitionOwnership() { zkClient.SlimLock.EnterWriteLock(); try { foreach (KeyValuePair <string, IDictionary <Partition, PartitionTopicInfo> > item in topicRegistry) { var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, item.Key); foreach (var partition in item.Value.Keys) { string znode = topicDirs.ConsumerOwnerDir + "/" + partition.Name; ZkUtils.DeletePath(zkClient, znode); if (Logger.IsDebugEnabled) { Logger.DebugFormat(CultureInfo.CurrentCulture, "Consumer {0} releasing {1}", this.consumerIdString, znode); } } topicRegistry.Remove(item.Key); } } finally { zkClient.SlimLock.ExitWriteLock(); } }
private long ResetConsumerOffsets(string topic, int partitionId) { long offset; switch (_config.AutoOffsetReset) { case OffsetRequest.SmallestTime: offset = OffsetRequest.EarliestTime; break; case OffsetRequest.LargestTime: offset = OffsetRequest.LatestTime; break; default: return(0); } var requestInfo = new Dictionary <string, List <PartitionOffsetRequestInfo> >(); requestInfo[topic] = new List <PartitionOffsetRequestInfo> { new PartitionOffsetRequestInfo(partitionId, offset, 1) }; var request = new OffsetRequest(requestInfo); OffsetResponse offsets = _simpleConsumer.GetOffsetsBefore(request); var topicDirs = new ZKGroupTopicDirs(_config.GroupId, topic); long offsetFound = offsets.ResponseMap[topic].First().Offsets[0]; Logger.InfoFormat("updating partition {0} with {1} offset {2}", partitionId, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsetFound); ZkUtils.UpdatePersistentPath(_zkClient, topicDirs.ConsumerOffsetDir + "/" + partitionId, offsetFound.ToString(CultureInfo.InvariantCulture)); return(offsetFound); }
internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount) { this.EnsuresNotDisposed(); Logger.InfoFormat(CultureInfo.CurrentCulture, "begin registering consumer {0} in ZK", consumerIdString); ZkUtils.CreateEphemeralPathExpectConflict(this.zkClient, dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString()); Logger.InfoFormat(CultureInfo.CurrentCulture, "end registering consumer {0} in ZK", consumerIdString); }
/// <summary>Pre-creating bookkeeper metadata path in zookeeper.</summary> /// <exception cref="System.IO.IOException"/> private void PrepareBookKeeperEnv() { // create bookie available path in zookeeper if it doesn't exists string zkAvailablePath = conf.Get(BkjmZkLedgersAvailablePath, BkjmZkLedgersAvailablePathDefault ); CountDownLatch zkPathLatch = new CountDownLatch(1); AtomicBoolean success = new AtomicBoolean(false); AsyncCallback.StringCallback callback = new _StringCallback_255(zkAvailablePath, success, zkPathLatch); ZkUtils.AsyncCreateFullPathOptimistic(zkc, zkAvailablePath, new byte[0], ZooDefs.Ids .OpenAclUnsafe, CreateMode.Persistent, callback, null); try { if (!zkPathLatch.Await(zkc.GetSessionTimeout(), TimeUnit.Milliseconds) || !success .Get()) { throw new IOException("Couldn't create bookie available path :" + zkAvailablePath + ", timed out " + zkc.GetSessionTimeout() + " millis"); } } catch (Exception e) { Sharpen.Thread.CurrentThread().Interrupt(); throw new IOException("Interrupted when creating the bookie available path : " + zkAvailablePath, e); } }
public void HandleDataChange(Events.ZooKeeperDataChangedEventArgs args) { int parsedLeader; string nodePath = args.Path; string nodeData = args.Data; Logger.Info("A partition leader or ISR list has been updated. Determining if rebalancing is necessary"); if (!ZkUtils.TryParsePartitionLeader(nodeData, out parsedLeader)) { Logger.Error("Skipping rebalancing. Failed to parse partition leader for path: " + nodePath + " from ZK node"); return; } else { if (!_partitionLeaderMap.ContainsKey(nodePath) || _partitionLeaderMap[nodePath] != parsedLeader) { string currentLeader = _partitionLeaderMap.ContainsKey(nodePath) ? _partitionLeaderMap[nodePath].ToString() : "null"; Logger.Info("Performing rebalancing. Leader value for path: " + nodePath + " has changed from " + currentLeader + " to " + parsedLeader); _partitionLeaderMap[nodePath] = parsedLeader; _rebalancer.AsyncRebalance(); } else { Logger.Info("Skipping rebalancing. Leader value for path: " + nodePath + " is " + parsedLeader.ToString() + " and has not changed"); } } }
/// <summary> /// Commits the offsets of all messages consumed so far. /// </summary> public void CommitOffsets() { this.EnsuresNotDisposed(); if (this.GetZkClient() == null) { return; } try { foreach (KeyValuePair <string, IDictionary <int, PartitionTopicInfo> > topic in topicRegistry) { var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic.Key); foreach (KeyValuePair <int, PartitionTopicInfo> partition in topic.Value) { var newOffset = partition.Value.ConsumeOffset; try { if (partition.Value.ConsumeOffsetValid) { // Save offsets unconditionally. Kafka's latestOffset for a particular topic-partition can go backward // if a follwer which is not fully caught up becomes a leader. We still need to save the conumed offsets even then. //skip only if we are trying to commit the same offset if (newOffset != partition.Value.CommitedOffset) { try { ZkUtils.UpdatePersistentPath(GetZkClient(), topicDirs.ConsumerOffsetDir + "/" + partition.Value.PartitionId, newOffset.ToString()); partition.Value.CommitedOffset = newOffset; } catch (Exception ex) { Logger.ErrorFormat("error in CommitOffsets UpdatePersistentPath : {0}", ex.FormatException()); } } } else { Logger.InfoFormat("Skip committing offset {0} for topic {1} because it is invalid (ZK session is disconnected)", newOffset, partition); } } catch (Exception ex) { Logger.WarnFormat("exception during CommitOffsets: {0}", ex.FormatException()); } if (Logger.IsDebugEnabled) { Logger.DebugFormat("Commited offset {0} for topic {1}", newOffset, partition); } } } } catch (Exception ex) { Logger.ErrorFormat("error in CommitOffsets : {0}", ex.FormatException()); } }
public override IDictionary <string, ISet <string> > GetConsumerThreadIdsPerTopic() { var wildcardTopics = ZkUtils.GetChildrenParentMayNotExist(this.ZkClient, ZkUtils.BrokerTopicsPath) .Where(x => TopicFilter.IsTopicAllowed(x)) .ToList(); return(this.MakeConsumerThreadIdsPerTopic(this.ConsumerIdString, wildcardTopics.ToDictionary(x => x, v => this.NumStreams))); }
private void DeletePartitionOwnershipFromZK(string topic, int partition) { var topicDirs = new ZKGroupTopicDirs(group, topic); var znode = topicDirs.ConsumerOwnerDir + "/" + partition; ZkUtils.DeletePath(this.parent.zkClient, znode); Logger.DebugFormat("Consumer {0} releasing {1}", consumerIdString, znode); }
internal static void Run(ProduceSimpleHelperOption produceOptions) { failedMessageCount = 0; successMessageCount = 0; sentBatchCount = 0; produceMessagePerPartition = new Dictionary <int, int>(); produceMessagePerPartitionExpect = new Dictionary <int, int>(); PrepareSentMessages(produceOptions); kafkaSimpleManagerConfig = new KafkaSimpleManagerConfiguration() { Zookeeper = produceOptions.Zookeeper, MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize, PartitionerClass = produceOptions.PartitionerClass }; kafkaSimpleManagerConfig.Verify(); producerConfigTemplate = new ProducerConfiguration( new List <BrokerConfiguration>() { }) //The Brokers will be replaced inside of KafkaSimpleManager { ForceToPartition = -1, PartitionerClass = kafkaSimpleManagerConfig.PartitionerClass, TotalNumPartitions = 0, RequiredAcks = produceOptions.RequiredAcks, AckTimeout = produceOptions.AckTimeout, SendTimeout = produceOptions.SendTimeout, ReceiveTimeout = produceOptions.ReceiveTimeout, CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()), BufferSize = produceOptions.BufferSize, SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize) }; using (ZooKeeperClient zkClient = new ZooKeeperClient(produceOptions.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); Dictionary <int, int[]> topicDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, produceOptions.Topic); // -2 by default or customized partitioner class. (you need specify PartitionerClass by -l) if (produceOptions.PartitionId == -2) { if (string.IsNullOrEmpty(kafkaSimpleManagerConfig.PartitionerClass)) { throw new ArgumentException("The partitioer class must not be empty if you want to send to partition by partitioner."); } //if (producerConfigTemplate.TotalNumPartitions <= 0) // throw new ArgumentException(string.Format("Please also specify the TotalNumPartitions if you want to send to partition by partitioner.")); ProduceByPartitionerClass(produceOptions, topicDataInZookeeper.Count); } else { ProduceToRandomOrSpecificPartition(produceOptions); } } }
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient) { var dirs = new ZKGroupDirs(group); var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1; string subscriptionPattern; IDictionary <string, int> topMap; try { var parsedJson = JObject.Parse(topicCountString); if (parsedJson != null) { var pattern = parsedJson.Get("pattern"); if (pattern != null) { subscriptionPattern = pattern.Value <string>(); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } var topMapObject = (IEnumerable <KeyValuePair <string, JToken> >)parsedJson.Get("subscription"); if (topMapObject != null) { topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value <int>()); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } catch (Exception e) { Logger.Error("error parsing consumer json string " + topicCountString, e); throw; } var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern); var hasBlackList = BlackListPattern.Equals(subscriptionPattern); if (topMap.Count == 0 || !(hasWhiteList || hasBlackList)) { return(new StaticTopicCount(consumerId, topMap)); } else { var regex = topMap.First().Key; var numStreams = topMap.First().Value; TopicFilter filter = hasWhiteList ? (TopicFilter) new Whitelist(regex) : new Blacklist(regex); return(new WildcardTopicCount(zkClient, consumerId, filter, numStreams)); } }
public static void UpdateConsumerOffset(ConsumerConfig config, string path, long offset) { var zkClient = new ZkClient( config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, config.ZooKeeper.ZkConnectionTimeoutMs, new ZkStringSerializer()); ZkUtils.UpdatePersistentPath(zkClient, path, offset.ToString(CultureInfo.InvariantCulture)); }
/// <summary> /// Commit offset of specified topic/partition. /// Only used when customer has strong requirement for reprocess messages as few as possible. /// </summary> /// <param name="topic"></param> /// <param name="partition"></param> /// <param name="offset"></param> /// <param name="setPosition">Indicates whether to set the fetcher's offset to the value committed. Default = true.</param> public void CommitOffset(string topic, int partition, long offset, bool setPosition = true) { this.EnsuresNotDisposed(); if (this.GetZkClient() == null) { return; } if (this.config.AutoCommit == true) { throw new ArgumentException(string.Format("When do commit offset with desired partition and offset, must set AutoCommit of ConsumerConfiguration as false!")); } try { IDictionary <int, PartitionTopicInfo> topicPartitionInfo = topicRegistry[topic]; var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic); PartitionTopicInfo partitionTopicInfo = topicPartitionInfo[partition]; if (partitionTopicInfo.ConsumeOffsetValid) { //Commit offset unconditionally. This would give consumes to decide which offset to read/skip //if (offset > partitionTopicInfo.CommitedOffset) try { ZkUtils.UpdatePersistentPath(GetZkClient(), topicDirs.ConsumerOffsetDir + "/" + partitionTopicInfo.PartitionId, offset.ToString()); partitionTopicInfo.CommitedOffset = offset; if (setPosition) { partitionTopicInfo.ConsumeOffset = offset; partitionTopicInfo.FetchOffset = offset; } } catch (Exception ex) { Logger.ErrorFormat("error in CommitOffsets UpdatePersistentPath : {0}", ex.FormatException()); } } else { Logger.InfoFormat( "Skip committing offset {0} for topic {1} because it is invalid (ZK session is disconnected)", offset, partitionTopicInfo); } if (Logger.IsDebugEnabled) { Logger.DebugFormat("Commited offset {0} for topic {1}", offset, partitionTopicInfo); } } catch (Exception ex) { Logger.ErrorFormat("exception during CommitOffsets: Topic:{0} Partition:{1} offset:{2} Exception:{3} ", topic, partition, offset, ex.FormatException()); } }
public static void CreateTopic( ZkClient zkClient, string topic, int partitions, int replicationFactor, Dictionary <string, string> topicConfig) { var brokerList = ZkUtils.GetSortedBrokerList(zkClient); var replicaAssigment = AssignReplicasToBrokers(brokerList, partitions, replicationFactor); CreateOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, replicaAssigment, topicConfig); }
public void StartWatchingTopicEvents() { var topicEventListener = new ZkTopicEventListener(this); ZkUtils.MakeSurePersistentPathExists(zkClient, ZkUtils.BrokerTopicsPath); this.zkClient.SubscribeStateChanges(new ZkSessionExpireListener(this, topicEventListener)); var topics = this.zkClient.SubscribeChildChanges(ZkUtils.BrokerTopicsPath, topicEventListener); // call to bootstrap topic list topicEventListener.HandleChildChange(ZkUtils.BrokerTopicsPath, topics); }
public static int?WaitUntilLeaderIsElectedOrChanged( ZkClient zkClient, string topic, int partition, long timeoutMs, int?oldLeaderOpt = null) { var leaderLock = new ReentrantLock(); var leaderExistsOrChanged = leaderLock.NewCondition(); if (oldLeaderOpt.HasValue == false) { Logger.InfoFormat("Waiting for leader to be elected for partition [{0},{1}]", topic, partition); } else { Logger.InfoFormat("Waiting for leader for partition [{0},{1}] to be changed from old leader {2}", topic, partition, oldLeaderOpt.Value); } leaderLock.Lock(); try { zkClient.SubscribeDataChanges(ZkUtils.GetTopicPartitionLeaderAndIsrPath(topic, partition), new LeaderExistsOrChangedListener(topic, partition, leaderLock, leaderExistsOrChanged, oldLeaderOpt, zkClient)); leaderExistsOrChanged.Await(TimeSpan.FromMilliseconds(timeoutMs)); // check if leader is elected var leader = ZkUtils.GetLeaderForPartition(zkClient, topic, partition); if (leader != null) { if (oldLeaderOpt.HasValue == false) { Logger.InfoFormat("Leader {0} is elected for partition [{1},{2}]", leader, topic, partition); } else { Logger.InfoFormat( "Leader for partition [{0},{1}] is changed from {2} to {3}", topic, partition, oldLeaderOpt.Value, leader); } } else { Logger.ErrorFormat("Timing out after {0} ms since leader is not elected for partition [{1},{2}]", timeoutMs, topic, partition); } return(leader); } finally { leaderLock.Unlock(); } }
public void SyncedRebalance() { lock (parent.rebalanceLock) { if (parent.isShuttingDown.Get()) { return; } else { for (var i = 1; i <= parent.Config.RebalanceMaxRetries; i++) { Logger.InfoFormat("begin rebalancing consumer {0} try #{1}", consumerIdString, i); var done = false; Cluster cluster = null; try { cluster = ZkUtils.GetCluster(parent.zkClient); done = this.Rebalance(cluster); } catch (Exception e) { /** occasionally, we may hit a ZK exception because the ZK state is changing while we are iterating. * For example, a ZK node can disappear between the time we get all children and the time we try to get * the value of a child. Just let this go since another rebalance will be triggered. **/ Logger.Info("Exception during rebalance", e); } Logger.InfoFormat("end rebalancing consumer {0} try #{1}", consumerIdString, i); if (done) { return; } else { /* Here the cache is at a risk of being stale. To take future rebalancing decisions correctly, we should * clear the cache */ Logger.Info("Rebalancing attempt failed. Clearing the cache before the next rebalancing operation is triggered"); } // stop all fetchers and clear all the queues to avoid Data duplication CloseFetchersForQueues(cluster, (IDictionary <string, IList <KafkaStream <TKey, TValue> > >)KafkaMessageAndMetadataStreams, parent.topicThreadIdAndQueues.Select(x => x.Value).ToList()); Thread.Sleep(parent.Config.RebalanceBackoffMs); } } } throw new ConsumerRebalanceFailedException( consumerIdString + " can't rebalance after " + parent.Config.RebalanceMaxRetries + " retries"); }
private void AddPartitionTopicInfo(ZKGroupTopicDirs topicDirs, string partition, string topic, string consumerThreadId) { var partitionId = int.Parse(partition); var partTopicInfoMap = topicRegistry[topic]; //find the leader for this partition var leaderOpt = ZkUtils.GetLeaderForPartition(zkClient, topic, partitionId); if (!leaderOpt.HasValue) { throw new NoBrokersForPartitionException( string.Format("No leader available for partitions {0} on topic {1}", partition, topic)); } Logger.InfoFormat("Leader for partition {0} for topic {1} is {2}", partition, topic, leaderOpt.Value); var leader = leaderOpt.Value; var znode = topicDirs.ConsumerOffsetDir + "/" + partition; var offsetCommitedString = zkClient.ReadData <string>(znode, true); //if first time starting a consumer, set the initial offset based on the config long offset = -1; long offsetCommited = -1; if (offsetCommitedString != null) { offsetCommited = long.Parse(offsetCommitedString); offset = offsetCommited + 1; } Logger.InfoFormat("Final offset {0} for topic {1} partition {2} OffsetCommited {3}" , offset, topic, partition, offsetCommited); var queue = queues[new Tuple <string, string>(topic, consumerThreadId)]; var partTopicInfo = new PartitionTopicInfo( topic, leader, partitionId, queue, offsetCommited, offset, config.FetchSize, offsetCommited); partTopicInfoMap[partitionId] = partTopicInfo; Logger.InfoFormat("{0} selected new offset {1}", partTopicInfo, offset); }
private bool ReflectPartitionOwnershipDecision( IDictionary <Tuple <string, int>, string> partitionOwnershipDecision) { var successfullyOwnedPartitions = new List <Tuple <string, int> >(); var partitionOwnershipSuccessful = partitionOwnershipDecision.Select( partitionOwner => { var topic = partitionOwner.Key.Item1; var partition = partitionOwner.Key.Item2; var consumerThreadId = partitionOwner.Value; var partitionOwnerPath = ZkUtils.GetConsumerPartitionOwnerPath(group, topic, partition); try { ZkUtils.CreateEphemeralPathExpectConflict( parent.zkClient, partitionOwnerPath, consumerThreadId); Logger.InfoFormat( "{0} successfully owner partition {1} for topic {2}", consumerThreadId, partition, topic); successfullyOwnedPartitions.Add(Tuple.Create(topic, partition)); return(true); } catch (ZkNodeExistsException) { // The node hasn't been deleted by the original owner. So wait a bit and retry. Logger.InfoFormat("waiting for the partition ownership to be deleted:" + partition); return(false); } }).ToList(); var hasPartitionOwnershipFailed = partitionOwnershipSuccessful.Aggregate( 0, (sum, decision) => (sum + (decision ? 0 : 1))); /* even if one of the partition ownership attempt has failed, return false */ if (hasPartitionOwnershipFailed > 0) { // remove all paths that we have owned in ZK foreach (var topicAndPartition in successfullyOwnedPartitions) { this.DeletePartitionOwnershipFromZK(topicAndPartition.Item1, topicAndPartition.Item2); } return(false); } return(true); }
internal WildcardStreamsHandler( ZookeeperConsumerConnector parent, TopicFilter topicFilter, int numStreams, IDecoder <TKey> keyDecoder, IDecoder <TValue> valueDecoder) { this.parent = parent; this.topicFilter = topicFilter; this.numStreams = numStreams; this.keyDecoder = keyDecoder; this.valueDecoder = valueDecoder; if (parent.messageStreamCreated.GetAndSet(true)) { throw new Exception("Each consumer connector can create message streams by filter at most once."); } this.wildcardQueuesAndStreams = Enumerable.Range(1, numStreams).Select(e => { var queue = new BlockingCollection <FetchedDataChunk>(this.parent.Config.QueuedMaxMessages); var stream = new KafkaStream <TKey, TValue>( queue, this.parent.Config.ConsumerTimeoutMs, keyDecoder, valueDecoder, this.parent.Config.ClientId); return(Tuple.Create(queue, stream)); }).ToList(); this.wildcardTopics = ZkUtils.GetChildrenParentMayNotExist(this.parent.zkClient, ZkUtils.BrokerTopicsPath) .Where(topicFilter.IsTopicAllowed) .ToList(); this.wildcardTopicCount = TopicCount.ConstructTopicCount( this.parent.consumerIdString, topicFilter, numStreams, this.parent.zkClient); this.dirs = new ZKGroupDirs(this.parent.Config.GroupId); this.parent.RegisterConsumerInZK(dirs, this.parent.consumerIdString, this.wildcardTopicCount); this.parent.ReinitializeConsumer(this.wildcardTopicCount, this.wildcardQueuesAndStreams); // Topic events will trigger subsequent synced rebalances. Logger.InfoFormat("Creating topic event watcher for topics {0}", topicFilter); this.parent.wildcardTopicWatcher = new ZookeeperTopicEventWatcher(this.parent.zkClient, this); }
private bool ReflectPartitionOwnershipDecision(string topic, string partition, string consumerThreadId) { var ownershipReflected = false; var topicDirs = new ZKGroupTopicDirs(config.GroupId, topic); var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition; try { Logger.InfoFormat("Consumer {0} will own partition {1} for topic {2}, will create ZK path {3}", consumerThreadId, partition, topic, partitionOwnerPath); try { zkClient.SlimLock.EnterWriteLock(); ZkUtils.CreateEphemeralPathExpectConflict(zkClient, partitionOwnerPath, consumerThreadId); Logger.InfoFormat( "Consumer {0} SUCC owned partition {1} for topic {2} . finish create ZK path {3} .", consumerThreadId, partition, topic, partitionOwnerPath); ownershipReflected = true; } catch (Exception ex) { Logger.InfoFormat( "Consumer {0} FAILED owned partition {1} for topic {2} . finish create ZK path {3} error:{4}.", consumerThreadId, partition, topic, partitionOwnerPath, ex.FormatException()); } finally { zkClient.SlimLock.ExitWriteLock(); } } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NODEEXISTS) { Logger.InfoFormat( "{0} failed to own partition {1} for topic {2}. Waiting for the partition owner to be deleted", consumerThreadId, partition, topic); ownershipReflected = false; } else { throw; } } return(ownershipReflected); }
public void ReleaseAllPartitionOwnerships() { Logger.Info("Releasing all partition ownerships"); var consumerIdString = GetConsumerIdString(); foreach (var item in topicRegistry) { var topic = item.Key; try { foreach (var partition in item.Value.Keys) { var partitionOwnerPath = ZkUtils.GetConsumerPartitionOwnerPath(config.GroupId, topic, partition.ToString()); Logger.InfoFormat("Consumer {0} will delete ZK path {1} topic:{2} partition:{3} ", consumerIdString, partitionOwnerPath, topic, partition); try { GetZkClient().SlimLock.EnterWriteLock(); ZkUtils.DeletePath(GetZkClient(), partitionOwnerPath); Logger.InfoFormat( "Consumer {0} SUCC delete ZK path {1} topic:{2} partition:{3} succsessfully.", consumerIdString, partitionOwnerPath, topic, partition); } catch (Exception ex) { Logger.ErrorFormat( "Consumer {0} FAILED delete ZK path {1} topic:{2} partition:{3} error:{4}.", consumerIdString, partitionOwnerPath, topic, partition, ex.FormatException()); } finally { GetZkClient().SlimLock.ExitWriteLock(); } } } catch (Exception ex) { Logger.ErrorFormat("error when call ZkUtils.DeletePath : {0}", ex.FormatException()); } } Logger.Info("Released all partition ownerships"); }
private bool ProcessPartition(ZKGroupTopicDirs topicDirs, string partition, string topic, string consumerThreadId) { var partitionOwnerPath = topicDirs.ConsumerOwnerDir + "/" + partition; try { ZkUtils.CreateEphemeralPathExpectConflict(zkClient, partitionOwnerPath, consumerThreadId); } catch (KeeperException.NodeExistsException) { //// The node hasn't been deleted by the original owner. So wait a bit and retry. Logger.InfoFormat(CultureInfo.CurrentCulture, "waiting for the partition ownership to be deleted: {0}", partition); return(false); } AddPartitionTopicInfo(topicDirs, partition, topic, consumerThreadId); return(true); }
internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount) { this.EnsuresNotDisposed(); Logger.InfoFormat("begin registering consumer {0} in ZK", consumerIdString); try { this.GetZkClient().SlimLock.EnterWriteLock(); ZkUtils.CreateEphemeralPathExpectConflict(this.GetZkClient(), dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString()); Logger.InfoFormat("successfully registering consumer {0} in ZK", consumerIdString); } catch (Exception ex) { Logger.ErrorFormat("error in RegisterConsumerInZk CreateEphemeralPathExpectConflict : {0}", ex.FormatException()); } finally { GetZkClient().SlimLock.ExitWriteLock(); } }
private static void WriteTopicPartitionAssignment( ZkClient zkClient, string topic, Dictionary <int, List <int> > replicaAssignment, bool update) { var zkPath = ZkUtils.GetTopicPath(topic); var jsonPartitonData = ZkUtils.ReplicaAssignmentZkData(replicaAssignment.ToDictionary(x => x.Key.ToString(), v => v.Value)); if (!update) { Logger.InfoFormat("Topic creation: {0}", jsonPartitonData); ZkUtils.CreatePersistentPath(zkClient, zkPath, jsonPartitonData); } else { Logger.InfoFormat("Topic update {0}" + jsonPartitonData); ZkUtils.UpdatePersistentPath(zkClient, zkPath, jsonPartitonData); } Logger.DebugFormat("Updated path {0} with {1} for replica assignment", zkPath, jsonPartitonData); }
public void AddProducers(ProducerConfiguration config) { var configuredBrokers = config.Brokers.Select(x => new Broker(x.BrokerId, x.Host, x.Port)); if (configuredBrokers.Any()) { configuredBrokers.ForEach(this.AddProducer); } else if (this.zkClient != null) { Logger.DebugFormat("Connecting to {0} for creating sync producers for all brokers in the cluster", config.ZooKeeper.ZkConnect); var brokers = ZkUtils.GetAllBrokersInCluster(this.zkClient); brokers.ForEach(this.AddProducer); } else { throw new IllegalStateException("No producers found from configuration and zk not setup."); } }
public static void CreateOrUpdateTopicPartitionAssignmentPathInZK( ZkClient zkClient, string topic, Dictionary <int, List <int> > partitionReplicaAssignment, Dictionary <string, string> config, bool update = false) { // Note: validation was omited var topicPath = ZkUtils.GetTopicPath(topic); if (!update && zkClient.Exists(topicPath)) { throw new KafkaException("Topic " + topic + " already exists."); } // write out the config if there is any, this isn't transactional with the partition assignments WriteTopicConfig(zkClient, topic, config); // create the partition assignment WriteTopicPartitionAssignment(zkClient, topic, partitionReplicaAssignment, update); }