public DataChange(ReentrantLock dataLock, ICondition dataExistsOrChanged, ZkClient zkClient) { Console.WriteLine("init data change..."); this.dataLock = dataLock; this.dataExistsOrChanged = dataExistsOrChanged; this.zkClient = zkClient; }
public static IDictionary <string, List <string> > GetConsumersPerTopic(ZkClient zkClient, string group) { var dirs = new ZKGroupDirs(group); var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir); var consumerPerTopicMap = new Dictionary <string, List <string> >(); foreach (var consumer in consumers) { var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient); foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic()) { var topic = topicAndConsumer.Key; var consumerThreadIdSet = topicAndConsumer.Value; foreach (var consumerThreadId in consumerThreadIdSet) { var curConsumers = consumerPerTopicMap.Get(topic); if (curConsumers != null) { curConsumers.Add(consumerThreadId); } else { consumerPerTopicMap[topic] = new List <string> { consumerThreadId }; } } } } consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList()); return(consumerPerTopicMap); }
protected ZooKeeperTestHarness() { this.zkConfig = new TempZookeeperConfig(TestZkUtils.ZookeeperPort); this.ZkConnect = TestZkUtils.ZookeeperConnect; this.Zookeeper = KafkaRunClassHelper.Run(KafkaRunClassHelper.ZookeeperMainClass, this.zkConfig.ConfigLocation); this.ZkClient = new ZkClient(this.ZkConnect, ZkSessionTimeout, ZkConnectionTimeout, new ZkStringSerializer()); }
public static Tuple <string, Stat> ReadData(ZkClient client, string path) { var stat = new Stat(); var dataString = client.ReadData <string>(path, stat); return(Tuple.Create(dataString, stat)); }
private static void AddChildrenToStringBuilder(ZkClient zkClient, Func<string, bool> pathFilter, int level, StringBuilder builder, string startPath) { var children = zkClient.GetChildren(startPath); foreach (var node in children) { string nestedPath; if (startPath.EndsWith("/")) { nestedPath = startPath + node; } else { nestedPath = startPath + "/" + node; } if (pathFilter(nestedPath)) { builder.Append(GetSpaces(level - 1) + "'-" + "+" + node + "\n"); AddChildrenToStringBuilder(zkClient, pathFilter, level + 1, builder, nestedPath); } else { builder.Append(GetSpaces(level - 1) + "'-" + "-" + node + " (contents hidden)\n"); } } }
/// <summary> /// Create an ephemeral node with the given path and data. /// Throw NodeExistException if node already exists. /// </summary> /// <param name="client"></param> /// <param name="path"></param> /// <param name="data"></param> public static void CreateEphemeralPathExpectConflict(ZkClient client, string path, string data) { try { CreateEphemeralPath(client, path, data); } catch (ZkNodeExistsException) { // this can happen when there is connection loss; make sure the Data is what we intend to write string storedData = null; try { storedData = ReadData(client, path).Item1; } catch (ZkNoNodeException) { // the node disappeared; treat as if node existed and let caller handles this } if (storedData == null || storedData != data) { Logger.InfoFormat("Conflict in {0} Data: {1}, stored Data: {2}", path, data, storedData); throw; } else { // otherwise, the creation succeeded, return normally Logger.InfoFormat("{0} exists with value {1} during connection loss", path, data); } } }
/// <summary> /// 获取没有在处理的待办任务 /// </summary> /// <param name="taskType"></param> /// <returns></returns> public async Task <Dictionary <string, List <string> > > ListUndoingTask() { var data = await ListAllTodoTask(); var result = new Dictionary <string, List <string> >(); foreach (var keyValuePair in data) { var inProgressTask = await ZkClient.ListChildren(ConstData.InProgressPath + "/" + keyValuePair.Key); var inProgress = new List <string>(); if (inProgressTask != null && inProgressTask.Count > 0) { inProgress = inProgressTask.Select(p => p.Item1).ToList(); } var todoTask = await ZkClient.ListChildren(ConstData.TodoTaskPath + "/" + keyValuePair.Key); var todo = new List <string>(); if (todoTask.Count > 0) { todo = todoTask.Select(p => p.Item1).ToList(); } if (todo.Count > 0) { result.Add(keyValuePair.Key, todo.Except(inProgress).ToList()); } } return(result); }
/// <summary> /// make sure a persistent path exists in ZK. Create the path if not exist. /// </summary> /// <param name="client"></param> /// <param name="path"></param> public static void MakeSurePersistentPathExists(ZkClient client, string path) { if (!client.Exists(path)) { client.CreatePersistent(path, true); // won't throw NoNodeException or NodeExistsException } }
public static IDictionary <string, IDictionary <int, List <int> > > GetPartitionAssignmentForTopics( ZkClient zkClient, IList <string> topics) { IDictionary <string, IDictionary <int, List <int> > > ret = new Dictionary <string, IDictionary <int, List <int> > >(); foreach (var topic in topics) { var jsonPartitionMap = ReadDataMaybeNull(zkClient, GetTopicPath(topic)).Item1; IDictionary <int, List <int> > partitionMap = new Dictionary <int, List <int> >(); if (jsonPartitionMap != null) { var m = JObject.Parse(jsonPartitionMap); var replicaMap = (IDictionary <string, JToken>)m.Get("partitions"); if (replicaMap != null) { partitionMap = replicaMap.ToDictionary( kvp => int.Parse(kvp.Key), kvp => kvp.Value.Values <int>().ToList()); } } Logger.DebugFormat("Partition map for /brokers/topics/{0} is {1}", topic, JObject.FromObject(partitionMap).ToString(Formatting.None)); ret[topic] = partitionMap; } return(ret); }
public WildcardTopicCount(ZkClient zkClient, string consumerIdString, TopicFilter topicFilter, int numStreams) { this.ZkClient = zkClient; this.ConsumerIdString = consumerIdString; this.TopicFilter = topicFilter; this.NumStreams = numStreams; }
public void CreateSimpleTopicsAndAwaitLeader(ZkClient zkClient, List <string> topics, int brokerId) { foreach (var topic in topics) { AdminUtils.CreateTopic(zkClient, topic, 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500); } }
public static string ToString(ZkClient zkClient, string startPath, Func<string, bool> pathFilter) { var level = 1; var builder = new StringBuilder("+ (" + startPath + ")"); builder.Append("\n"); AddChildrenToStringBuilder(zkClient, pathFilter, level, builder, startPath); return builder.ToString(); }
public List <Tuple <string, string> > GetZKChildrenValues(string path) { var children = ZkClient.GetChildren(path).OrderBy(x => x).ToList(); return (children.Select(partition => Tuple.Create(partition, ZkClient.ReadData <string>(path + "/" + partition))) .ToList()); }
public LeaderExistsOrChangedListener(string topic, int partition, ReentrantLock leaderLock, ICondition leaderExistsOrChanged, int?oldLeaderOpt, ZkClient zkClient) { this.topic = topic; this.partition = partition; this.leaderLock = leaderLock; this.leaderExistsOrChanged = leaderExistsOrChanged; this.oldLeaderOpt = oldLeaderOpt; this.zkClient = zkClient; }
public static List<Broker> GetAllBrokersInCluster(ZkClient zkClient) { var brokerIds = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath).OrderBy(x => x).ToList(); return brokerIds.Select(int.Parse) .Select(id => GetBrokerInfo(zkClient, id)) .Where(x => x != null) .ToList(); }
public void ConnectZk() { Logger.InfoFormat("Connecting to zookeeper instance at " + this.Config.ZooKeeper.ZkConnect); this.zkClient = new ZkClient( this.Config.ZooKeeper.ZkConnect, this.Config.ZooKeeper.ZkSessionTimeoutMs, this.Config.ZooKeeper.ZkConnectionTimeoutMs, new ZkStringSerializer()); }
/// <summary> /// create the parent path /// </summary> /// <param name="client"></param> /// <param name="path"></param> private static void CreateParentPath(ZkClient client, string path) { var parentDir = path.Substring(0, path.LastIndexOf('/')); if (parentDir.Length != 0) { client.CreatePersistent(parentDir, true); } }
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient) { var dirs = new ZKGroupDirs(group); var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1; string subscriptionPattern; IDictionary<string, int> topMap; try { var parsedJson = JObject.Parse(topicCountString); if (parsedJson != null) { var pattern = parsedJson.Get("pattern"); if (pattern != null) { subscriptionPattern = pattern.Value<string>(); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } var topMapObject = (IEnumerable<KeyValuePair<string, JToken>>)parsedJson.Get("subscription"); if (topMapObject != null) { topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value<int>()); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } catch (Exception e) { Logger.Error("error parsing consumer json string " + topicCountString, e); throw; } var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern); var hasBlackList = BlackListPattern.Equals(subscriptionPattern); if (topMap.Count == 0 || !(hasWhiteList || hasBlackList)) { return new StaticTopicCount(consumerId, topMap); } else { var regex = topMap.First().Key; var numStreams = topMap.First().Value; TopicFilter filter = hasWhiteList ? (TopicFilter)new Whitelist(regex) : new Blacklist(regex); return new WildcardTopicCount(zkClient, consumerId, filter, numStreams); } }
public void WaitForServersToSettle() { foreach (TempKafkaConfig config in this.Configs) { if (!ZkClient.WaitUntilExists(ZkUtils.BrokerIdsPath + "/" + config.BrokerId, TimeSpan.FromSeconds(5))) { throw new Exception("Timeout on waiting for broker " + config.BrokerId + " to settle"); } } }
public static void UpdateConsumerOffset(ConsumerConfig config, string path, long offset) { var zkClient = new ZkClient( config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, config.ZooKeeper.ZkConnectionTimeoutMs, new ZkStringSerializer()); ZkUtils.UpdatePersistentPath(zkClient, path, offset.ToString(CultureInfo.InvariantCulture)); }
public static List <Broker> GetAllBrokersInCluster(ZkClient zkClient) { var brokerIds = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath).OrderBy(x => x).ToList(); return (brokerIds.Select(int.Parse) .Select(id => GetBrokerInfo(zkClient, id)) .Where(x => x != null) .ToList()); }
public static int?GetLeaderForPartition(ZkClient zkClient, string topic, int partition) { var leaderAndIsrOpt = ReadDataMaybeNull(zkClient, GetTopicPartitionLeaderAndIsrPath(topic, partition)).Item1; if (leaderAndIsrOpt != null) { return(JObject.Parse(leaderAndIsrOpt).SelectToken("leader").Value <int>()); } return(null); }
public static IList <string> GetChildrenParentMayNotExist(ZkClient client, string path) { try { return(client.GetChildren(path)); } catch (ZkNoNodeException) { return(null); } }
public static void CreateTopic( ZkClient zkClient, string topic, int partitions, int replicationFactor, Dictionary<string, string> topicConfig) { var brokerList = ZkUtils.GetSortedBrokerList(zkClient); var replicaAssigment = AssignReplicasToBrokers(brokerList, partitions, replicationFactor); CreateOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, replicaAssigment, topicConfig); }
/// <summary> /// Create an ephemeral node with the given path and data. Create parents if necessary. /// </summary> /// <param name="client"></param> /// <param name="path"></param> /// <param name="data"></param> private static void CreateEphemeralPath(ZkClient client, string path, string data) { try { client.CreateEphemeral(path, data); } catch (ZkNoNodeException) { CreateParentPath(client, path); client.CreateEphemeral(path, data); } }
static void Main(string[] args) { ZkConnection zkConnection = new ZkConnection("192.168.60.6:2181"); ZkClient zkClient = new ZkClient(ZkConnect, ZkSessionTimeout, ZkConnectionTimeout, new ZkStringSerializer()); var dataLock = new ReentrantLock(); var dataExistsOrChanged = dataLock.NewCondition(); zkClient.SubscribeDataChanges("/testyy", new DataChange(dataLock, dataExistsOrChanged, zkClient)); //dataExistsOrChanged.Await(TimeSpan.FromMilliseconds(1000)); Console.Read(); }
public static void CreateTopic( ZkClient zkClient, string topic, int partitions, int replicationFactor, Dictionary <string, string> topicConfig) { var brokerList = ZkUtils.GetSortedBrokerList(zkClient); var replicaAssigment = AssignReplicasToBrokers(brokerList, partitions, replicationFactor); CreateOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, replicaAssigment, topicConfig); }
public ConsumerFetcherManager(string consumerIdString, ConsumerConfig config, ZkClient zkClient) : base( string.Format("ConsumerFetcherManager-{0}", DateTimeHelper.CurrentTimeMilis()), config.ClientId, config.NumConsumerFetchers) { this.NoLeaderPartitionSet = new HashSet<TopicAndPartition>(); this.consumerIdString = consumerIdString; this.config = config; this.zkClient = zkClient; this.@lock = new ReentrantLock(); this.cond = [email protected](); }
/// <summary> /// Create an persistent node with the given path and data. Create parents if necessary. /// </summary> public static void CreatePersistentPath(ZkClient client, string path, string data) { try { client.CreatePersistent(path, data); } catch (ZkNoNodeException e) { CreateParentPath(client, path); client.CreatePersistent(path, data); } }
public ConsumerFetcherManager(string consumerIdString, ConsumerConfig config, ZkClient zkClient) : base( string.Format("ConsumerFetcherManager-{0}", DateTimeHelper.CurrentTimeMilis()), config.ClientId, config.NumConsumerFetchers) { this.NoLeaderPartitionSet = new HashSet <TopicAndPartition>(); this.consumerIdString = consumerIdString; this.config = config; this.zkClient = zkClient; this.@lock = new ReentrantLock(); this.cond = [email protected](); }
public static void MaybeDeletePath(string zkUrl, string dir) { try { var zk = new ZkClient(zkUrl, 30 * 1000, 30 * 1000, new ZkStringSerializer()); zk.DeleteRecursive(dir); zk.Dispose(); } catch { // swallow } }
public static bool DeletePath(ZkClient client, string path) { try { return(client.Delete(path)); } catch (ZkNoNodeException) { // this can happen during a connection loss event, return normally Logger.InfoFormat("{0} deleted during connection loss; This is ok. ", path); return(false); } }
/// <summary> /// This API takes in a broker id, queries zookeeper for the broker metadata and returns the metadata for that broker /// or throws an exception if the broker dies before the query to zookeeper finishes /// </summary> /// <param name="zkClient">The zookeeper client connection</param> /// <param name="brokerId">The broker id</param> /// <returns>An optional Broker object encapsulating the broker metadata</returns> public static Broker GetBrokerInfo(ZkClient zkClient, int brokerId) { var brokerInfo = ReadDataMaybeNull(zkClient, BrokerIdsPath + "/" + brokerId); if (brokerInfo != null) { return(Broker.CreateBroker(brokerId, brokerInfo.Item1)); } else { return(null); } }
public static Cluster GetCluster(ZkClient zkClient) { var cluster = new Cluster(); var nodes = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath); foreach (var node in nodes) { var brokerZkString = ReadData(zkClient, BrokerIdsPath + "/" + node).Item1; cluster.Add(Broker.CreateBroker(int.Parse(node), brokerZkString)); } return(cluster); }
public static int?WaitUntilLeaderIsElectedOrChanged( ZkClient zkClient, string topic, int partition, long timeoutMs, int?oldLeaderOpt = null) { var leaderLock = new ReentrantLock(); var leaderExistsOrChanged = leaderLock.NewCondition(); if (oldLeaderOpt.HasValue == false) { Logger.InfoFormat("Waiting for leader to be elected for partition [{0},{1}]", topic, partition); } else { Logger.InfoFormat("Waiting for leader for partition [{0},{1}] to be changed from old leader {2}", topic, partition, oldLeaderOpt.Value); } leaderLock.Lock(); try { zkClient.SubscribeDataChanges(ZkUtils.GetTopicPartitionLeaderAndIsrPath(topic, partition), new LeaderExistsOrChangedListener(topic, partition, leaderLock, leaderExistsOrChanged, oldLeaderOpt, zkClient)); leaderExistsOrChanged.Await(TimeSpan.FromMilliseconds(timeoutMs)); // check if leader is elected var leader = ZkUtils.GetLeaderForPartition(zkClient, topic, partition); if (leader != null) { if (oldLeaderOpt.HasValue == false) { Logger.InfoFormat("Leader {0} is elected for partition [{1},{2}]", leader, topic, partition); } else { Logger.InfoFormat( "Leader for partition [{0},{1}] is changed from {2} to {3}", topic, partition, oldLeaderOpt.Value, leader); } } else { Logger.ErrorFormat("Timing out after {0} ms since leader is not elected for partition [{1},{2}]", timeoutMs, topic, partition); } return(leader); } finally { leaderLock.Unlock(); } }
public static Tuple <string, Stat> ReadDataMaybeNull(ZkClient client, string path) { var stat = new Stat(); try { var obj = client.ReadData <string>(path, stat); return(Tuple.Create(obj, stat)); } catch (ZkNoNodeException) { return(Tuple.Create((string)null, stat)); } }
public void TestEphemeralNodeCleanup() { var zkClient = new ZkClient(ZkConnect, ZkSessionTimeoutMs, ZkConnectionTimeout, new ZkStringSerializer()); try { ZkUtils.CreateEphemeralPathExpectConflict(zkClient, "/tmp/zktest", "node created"); } catch { // ok } var testData = ZkUtils.ReadData(this.ZkClient, "/tmp/zktest").Item1; Assert.NotNull(testData); zkClient.Dispose(); zkClient = new ZkClient(ZkConnect, ZkSessionTimeoutMs, ZkConnectionTimeout, new ZkStringSerializer()); var nodeExists = ZkUtils.PathExists(zkClient, "/tmp/zktest"); Assert.False(nodeExists); }
public static void CreateOrUpdateTopicPartitionAssignmentPathInZK( ZkClient zkClient, string topic, Dictionary<int, List<int>> partitionReplicaAssignment, Dictionary<string, string> config, bool update = false) { // Note: validation was omited var topicPath = ZkUtils.GetTopicPath(topic); if (!update && zkClient.Exists(topicPath)) { throw new KafkaException("Topic " + topic +" already exists."); } // write out the config if there is any, this isn't transactional with the partition assignments WriteTopicConfig(zkClient, topic, config); // create the partition assignment WriteTopicPartitionAssignment(zkClient, topic, partitionReplicaAssignment, update); }
public ZookeeperTopicEventWatcher(ZkClient zkClient, ITopicEventHandler<string> eventHandler) { this.zkClient = zkClient; this.eventHandler = eventHandler; this.StartWatchingTopicEvents(); }
/// <summary> /// Create an ephemeral node with the given path and data. /// Throw NodeExistsException if node already exists. /// Handles the following ZK session timeout b_u_g /// /// https://issues.apache.org/jira/browse/ZOOKEEPER-1740 /// /// Upon receiving a NodeExistsException, read the data from the conflicted path and /// trigger the checker function comparing the read data and the expected data, /// If the checker function returns true then the above b_u_g might be encountered, back off and retry; /// otherwise re-throw the exception /// </summary> /// <param name="zkClient"></param> /// <param name="path"></param> /// <param name="data"></param> /// <param name="expectedCallerData"></param> /// <param name="checker"></param> /// <param name="backoffTime"></param> public static void CreateEphemeralPathExpectConflictHandleZKBug( ZkClient zkClient, string path, string data, object expectedCallerData, Func<string, object, bool> checker, int backoffTime) { while (true) { try { CreateEphemeralPathExpectConflict(zkClient, path, data); return; } catch (ZkNodeExistsException) { // An ephemeral node may still exist even after its corresponding session has expired // due to a Zookeeper ug, in this case we need to retry writing until the previous node is deleted // and hence the write succeeds without ZkNodeExistsException var writtenData = ReadDataMaybeNull(zkClient, path).Item1; if (writtenData != null) { if (checker(writtenData, expectedCallerData)) { Logger.InfoFormat( "I wrote this conflicted ephemeral node [{0}] at {1} a while back in a different session, " + "hence I will backoff for this node to be deleted by Zookeeper and retry", data, path); Thread.Sleep(backoffTime); } else { throw; } } else { // the node disappeared; retry creating the ephemeral node immediately } } } }
public static int? GetLeaderForPartition(ZkClient zkClient, string topic, int partition) { var leaderAndIsrOpt = ReadDataMaybeNull(zkClient, GetTopicPartitionLeaderAndIsrPath(topic, partition)).Item1; if (leaderAndIsrOpt != null) { return JObject.Parse(leaderAndIsrOpt).SelectToken("leader").Value<int>(); } return null; }
/// <summary> /// Update the value of a persistent node with the given path and data. /// create parrent directory if necessary. Never throw NodeExistException. /// Return the updated path zkVersion /// </summary> /// <param name="client"></param> /// <param name="path"></param> /// <param name="data"></param> public static void UpdatePersistentPath(ZkClient client, string path, string data) { try { client.WriteData(path, data); } catch (ZkNoNodeException) { CreateParentPath(client, path); try { client.CreatePersistent(path, data); } catch (ZkNodeExistsException) { client.WriteData(path, data); } } }
public static Tuple<string, Stat> ReadDataMaybeNull(ZkClient client, string path) { var stat = new Stat(); try { var obj = client.ReadData<string>(path, stat); return Tuple.Create(obj, stat); } catch (ZkNoNodeException) { return Tuple.Create((string)null, stat); } }
public static IDictionary<string, List<string>> GetConsumersPerTopic(ZkClient zkClient, string group) { var dirs = new ZKGroupDirs(group); var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir); var consumerPerTopicMap = new Dictionary<string, List<string>>(); // Consumers are named by their group Id plus the machine name and an identifier suffix. In this context, // we are asked to retrieve the consumers for the `group` argument, so we discard any group in /consumers/ids // that is not prefixed with the group id (filter using Where Enumerable.Where extension method) foreach (var consumer in consumers.Where(c => c.StartsWith(group))) { var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient); foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic()) { var topic = topicAndConsumer.Key; var consumerThreadIdSet = topicAndConsumer.Value; foreach (var consumerThreadId in consumerThreadIdSet) { var curConsumers = consumerPerTopicMap.Get(topic); if (curConsumers != null) { curConsumers.Add(consumerThreadId); } else { consumerPerTopicMap[topic] = new List<string> { consumerThreadId }; } } } } consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList()); return consumerPerTopicMap; }
public static IDictionary<string, IDictionary<int, List<int>>> GetPartitionAssignmentForTopics( ZkClient zkClient, IList<string> topics) { IDictionary<string, IDictionary<int, List<int>>> ret = new Dictionary<string, IDictionary<int, List<int>>>(); foreach (var topic in topics) { var jsonPartitionMap = ReadDataMaybeNull(zkClient, GetTopicPath(topic)).Item1; IDictionary<int, List<int>> partitionMap = new Dictionary<int, List<int>>(); if (jsonPartitionMap != null) { var m = JObject.Parse(jsonPartitionMap); var replicaMap = (IDictionary<string, JToken>)m.Get("partitions"); if (replicaMap != null) { partitionMap = replicaMap.ToDictionary( kvp => int.Parse(kvp.Key), kvp => kvp.Value.Values<int>().ToList()); } } Logger.DebugFormat("Partition map for /brokers/topics/{0} is {1}", topic, JObject.FromObject(partitionMap).ToString(Formatting.None)); ret[topic] = partitionMap; } return ret; }
public static Cluster GetCluster(ZkClient zkClient) { var cluster = new Cluster(); var nodes = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath); foreach (var node in nodes) { var brokerZkString = ReadData(zkClient, BrokerIdsPath + "/" + node).Item1; cluster.Add(Broker.CreateBroker(int.Parse(node), brokerZkString)); } return cluster; }
public static bool PathExists(ZkClient client, string path) { return client.Exists(path); }
public static IList<string> GetChildrenParentMayNotExist(ZkClient client, string path) { try { return client.GetChildren(path); } catch (ZkNoNodeException) { return null; } }
public static List<string> GetChildren(ZkClient zkClient, string path) { return zkClient.GetChildren(path); }
public void TestLeaderSelectionForPartition() { var zkClient = new ZkClient(this.zookeeperConnect, 6000, 30000, new ZkStringSerializer()); // create topic topic1 with 1 partition on broker 0 AdminUtils.CreateTopic(zkClient, Topic, 1, 1, new Dictionary<string, string>()); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 3000); var sentMessages1 = this.SendMessages( Configs.First(), nMessages, "batch1", CompressionCodecs.NoCompressionCodec, 1); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000); // create a consuemr var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1); var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1); var topicMessageStreams1 = zkConsumerConnector1.CreateMessageStreams( new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); var topicRegistry = zkConsumerConnector1.TopicRegistry; Assert.Equal(1, topicRegistry.Select(x => x.Key).Count()); Assert.Equal(Topic, topicRegistry.Select(x => x.Key).First()); var topicsAndPartitionsInRegistry = topicRegistry.Select(x => Tuple.Create(x.Key, x.Value.Select(p => p.Value))).ToList(); var brokerPartition = topicsAndPartitionsInRegistry.First().Item2.First(); Assert.Equal(0, brokerPartition.PartitionId); // also check partition ownership var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir); var expected_1 = new List<Tuple<string, string>> { Tuple.Create("0", "group1_consumer1-0"), }; Assert.Equal(expected_1, actual_1); var receivedMessages1 = this.GetMessages(nMessages, topicMessageStreams1); Assert.Equal(sentMessages1, receivedMessages1); zkConsumerConnector1.Shutdown(); zkClient.Dispose(); }
public static WildcardTopicCount ConstructTopicCount( string consumerIdString, TopicFilter filter, int numStream, ZkClient zkClient) { return new WildcardTopicCount(zkClient, consumerIdString, filter, numStream); }
/// <summary> /// This API takes in a broker id, queries zookeeper for the broker metadata and returns the metadata for that broker /// or throws an exception if the broker dies before the query to zookeeper finishes /// </summary> /// <param name="zkClient">The zookeeper client connection</param> /// <param name="brokerId">The broker id</param> /// <returns>An optional Broker object encapsulating the broker metadata</returns> public static Broker GetBrokerInfo(ZkClient zkClient, int brokerId) { var brokerInfo = ReadDataMaybeNull(zkClient, BrokerIdsPath + "/" + brokerId); if (brokerInfo != null) { return Broker.CreateBroker(brokerId, brokerInfo.Item1); } else { return null; } }
public static Tuple<string, Stat> ReadData(ZkClient client, string path) { var stat = new Stat(); var dataString = client.ReadData<string>(path, stat); return Tuple.Create(dataString, stat); }
public LeaderExistsOrChangedListener(string topic, int partition, ReentrantLock leaderLock, ICondition leaderExistsOrChanged, int? oldLeaderOpt, ZkClient zkClient) { this.topic = topic; this.partition = partition; this.leaderLock = leaderLock; this.leaderExistsOrChanged = leaderExistsOrChanged; this.oldLeaderOpt = oldLeaderOpt; this.zkClient = zkClient; }
public static List<int> GetSortedBrokerList(ZkClient zkClient) { return GetChildren(zkClient, BrokerIdsPath).Select(int.Parse).OrderBy(x => x).ToList(); }
public static bool DeletePath(ZkClient client, string path) { try { return client.Delete(path); } catch (ZkNoNodeException) { // this can happen during a connection loss event, return normally Logger.InfoFormat("{0} deleted during connection loss; This is ok. ", path); return false; } }