Esempio n. 1
0
 public DataChange(ReentrantLock dataLock, ICondition dataExistsOrChanged, ZkClient zkClient)
 {
     Console.WriteLine("init data change...");
     this.dataLock            = dataLock;
     this.dataExistsOrChanged = dataExistsOrChanged;
     this.zkClient            = zkClient;
 }
        public static IDictionary <string, List <string> > GetConsumersPerTopic(ZkClient zkClient, string group)
        {
            var dirs                = new ZKGroupDirs(group);
            var consumers           = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir);
            var consumerPerTopicMap = new Dictionary <string, List <string> >();

            foreach (var consumer in consumers)
            {
                var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient);
                foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic())
                {
                    var topic = topicAndConsumer.Key;
                    var consumerThreadIdSet = topicAndConsumer.Value;
                    foreach (var consumerThreadId in consumerThreadIdSet)
                    {
                        var curConsumers = consumerPerTopicMap.Get(topic);
                        if (curConsumers != null)
                        {
                            curConsumers.Add(consumerThreadId);
                        }
                        else
                        {
                            consumerPerTopicMap[topic] = new List <string> {
                                consumerThreadId
                            };
                        }
                    }
                }
            }

            consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList());

            return(consumerPerTopicMap);
        }
 protected ZooKeeperTestHarness()
 {
     this.zkConfig = new TempZookeeperConfig(TestZkUtils.ZookeeperPort);
     this.ZkConnect = TestZkUtils.ZookeeperConnect;
     this.Zookeeper = KafkaRunClassHelper.Run(KafkaRunClassHelper.ZookeeperMainClass, this.zkConfig.ConfigLocation); 
     this.ZkClient = new ZkClient(this.ZkConnect, ZkSessionTimeout, ZkConnectionTimeout, new ZkStringSerializer());
 }
        public static Tuple <string, Stat> ReadData(ZkClient client, string path)
        {
            var stat       = new Stat();
            var dataString = client.ReadData <string>(path, stat);

            return(Tuple.Create(dataString, stat));
        }
Esempio n. 5
0
        private static void AddChildrenToStringBuilder(ZkClient zkClient, Func<string, bool> pathFilter, int level, StringBuilder builder, string startPath)
        {
            var children = zkClient.GetChildren(startPath);
            foreach (var node in children) 
            {
                string nestedPath;
                if (startPath.EndsWith("/")) 
                {
                    nestedPath = startPath + node;
                } 
                else
                {
                    nestedPath = startPath + "/" + node;
                }

                if (pathFilter(nestedPath)) 
                {
                    builder.Append(GetSpaces(level - 1) + "'-" + "+" + node + "\n");
                    AddChildrenToStringBuilder(zkClient, pathFilter, level + 1, builder, nestedPath);
                }
                else
                {
                    builder.Append(GetSpaces(level - 1) + "'-" + "-" + node + " (contents hidden)\n");
                }
            }
        }
Esempio n. 6
0
 protected ZooKeeperTestHarness()
 {
     this.zkConfig  = new TempZookeeperConfig(TestZkUtils.ZookeeperPort);
     this.ZkConnect = TestZkUtils.ZookeeperConnect;
     this.Zookeeper = KafkaRunClassHelper.Run(KafkaRunClassHelper.ZookeeperMainClass, this.zkConfig.ConfigLocation);
     this.ZkClient  = new ZkClient(this.ZkConnect, ZkSessionTimeout, ZkConnectionTimeout, new ZkStringSerializer());
 }
        /// <summary>
        /// Create an ephemeral node with the given path and data.
        /// Throw NodeExistException if node already exists.
        /// </summary>
        /// <param name="client"></param>
        /// <param name="path"></param>
        /// <param name="data"></param>
        public static void CreateEphemeralPathExpectConflict(ZkClient client, string path, string data)
        {
            try
            {
                CreateEphemeralPath(client, path, data);
            }
            catch (ZkNodeExistsException)
            {
                // this can happen when there is connection loss; make sure the Data is what we intend to write
                string storedData = null;
                try
                {
                    storedData = ReadData(client, path).Item1;
                }
                catch (ZkNoNodeException)
                {
                    // the node disappeared; treat as if node existed and let caller handles this
                }

                if (storedData == null || storedData != data)
                {
                    Logger.InfoFormat("Conflict in {0} Data: {1}, stored Data: {2}", path, data, storedData);
                    throw;
                }
                else
                {
                    // otherwise, the creation succeeded, return normally
                    Logger.InfoFormat("{0} exists with value {1} during connection loss", path, data);
                }
            }
        }
Esempio n. 8
0
        /// <summary>
        /// 获取没有在处理的待办任务
        /// </summary>
        /// <param name="taskType"></param>
        /// <returns></returns>
        public async Task <Dictionary <string, List <string> > > ListUndoingTask()
        {
            var data = await ListAllTodoTask();

            var result = new Dictionary <string, List <string> >();

            foreach (var keyValuePair in data)
            {
                var inProgressTask = await ZkClient.ListChildren(ConstData.InProgressPath + "/" + keyValuePair.Key);

                var inProgress = new List <string>();
                if (inProgressTask != null && inProgressTask.Count > 0)
                {
                    inProgress = inProgressTask.Select(p => p.Item1).ToList();
                }
                var todoTask = await ZkClient.ListChildren(ConstData.TodoTaskPath + "/" + keyValuePair.Key);

                var todo = new List <string>();
                if (todoTask.Count > 0)
                {
                    todo = todoTask.Select(p => p.Item1).ToList();
                }
                if (todo.Count > 0)
                {
                    result.Add(keyValuePair.Key, todo.Except(inProgress).ToList());
                }
            }
            return(result);
        }
 /// <summary>
 /// make sure a persistent path exists in ZK. Create the path if not exist.
 /// </summary>
 /// <param name="client"></param>
 /// <param name="path"></param>
 public static void MakeSurePersistentPathExists(ZkClient client, string path)
 {
     if (!client.Exists(path))
     {
         client.CreatePersistent(path, true); // won't throw NoNodeException or NodeExistsException
     }
 }
        public static IDictionary <string, IDictionary <int, List <int> > > GetPartitionAssignmentForTopics(
            ZkClient zkClient, IList <string> topics)
        {
            IDictionary <string, IDictionary <int, List <int> > > ret = new Dictionary <string, IDictionary <int, List <int> > >();

            foreach (var topic in topics)
            {
                var jsonPartitionMap = ReadDataMaybeNull(zkClient, GetTopicPath(topic)).Item1;
                IDictionary <int, List <int> > partitionMap = new Dictionary <int, List <int> >();
                if (jsonPartitionMap != null)
                {
                    var m          = JObject.Parse(jsonPartitionMap);
                    var replicaMap = (IDictionary <string, JToken>)m.Get("partitions");
                    if (replicaMap != null)
                    {
                        partitionMap = replicaMap.ToDictionary(
                            kvp => int.Parse(kvp.Key), kvp => kvp.Value.Values <int>().ToList());
                    }
                }

                Logger.DebugFormat("Partition map for /brokers/topics/{0} is {1}", topic, JObject.FromObject(partitionMap).ToString(Formatting.None));
                ret[topic] = partitionMap;
            }

            return(ret);
        }
 public WildcardTopicCount(ZkClient zkClient, string consumerIdString, TopicFilter topicFilter, int numStreams)
 {
     this.ZkClient         = zkClient;
     this.ConsumerIdString = consumerIdString;
     this.TopicFilter      = topicFilter;
     this.NumStreams       = numStreams;
 }
 public void CreateSimpleTopicsAndAwaitLeader(ZkClient zkClient, List <string> topics, int brokerId)
 {
     foreach (var topic in topics)
     {
         AdminUtils.CreateTopic(zkClient, topic, 1, 1, new Dictionary <string, string>());
         TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500);
     }
 }
Esempio n. 13
0
 public static string ToString(ZkClient zkClient, string startPath, Func<string, bool> pathFilter) 
 {
     var level = 1;
     var builder = new StringBuilder("+ (" + startPath + ")");
     builder.Append("\n");
     AddChildrenToStringBuilder(zkClient, pathFilter, level, builder, startPath);
     return builder.ToString();
 }
        public List <Tuple <string, string> > GetZKChildrenValues(string path)
        {
            var children = ZkClient.GetChildren(path).OrderBy(x => x).ToList();

            return
                (children.Select(partition => Tuple.Create(partition, ZkClient.ReadData <string>(path + "/" + partition)))
                 .ToList());
        }
 public LeaderExistsOrChangedListener(string topic, int partition, ReentrantLock leaderLock, ICondition leaderExistsOrChanged, int?oldLeaderOpt, ZkClient zkClient)
 {
     this.topic                 = topic;
     this.partition             = partition;
     this.leaderLock            = leaderLock;
     this.leaderExistsOrChanged = leaderExistsOrChanged;
     this.oldLeaderOpt          = oldLeaderOpt;
     this.zkClient              = zkClient;
 }
Esempio n. 16
0
 public static List<Broker> GetAllBrokersInCluster(ZkClient zkClient)
 {
     var brokerIds = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath).OrderBy(x => x).ToList();
     return
         brokerIds.Select(int.Parse)
                  .Select(id => GetBrokerInfo(zkClient, id))
                  .Where(x => x != null)
                  .ToList();
 }
Esempio n. 17
0
 public void ConnectZk()
 {
     Logger.InfoFormat("Connecting to zookeeper instance at " + this.Config.ZooKeeper.ZkConnect);
     this.zkClient = new ZkClient(
         this.Config.ZooKeeper.ZkConnect,
         this.Config.ZooKeeper.ZkSessionTimeoutMs,
         this.Config.ZooKeeper.ZkConnectionTimeoutMs,
         new ZkStringSerializer());
 }
        /// <summary>
        ///  create the parent path
        /// </summary>
        /// <param name="client"></param>
        /// <param name="path"></param>
        private static void CreateParentPath(ZkClient client, string path)
        {
            var parentDir = path.Substring(0, path.LastIndexOf('/'));

            if (parentDir.Length != 0)
            {
                client.CreatePersistent(parentDir, true);
            }
        }
Esempio n. 19
0
        public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient)
        {
            var dirs = new ZKGroupDirs(group);
            var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1;
            string subscriptionPattern;
            IDictionary<string, int> topMap;
            try
            {
                var parsedJson = JObject.Parse(topicCountString);
                if (parsedJson != null)
                {
                    var pattern = parsedJson.Get("pattern");
                    if (pattern != null)
                    {
                        subscriptionPattern = pattern.Value<string>();
                    }
                    else
                    {
                        throw new KafkaException("error constructing TopicCount:" + topicCountString);
                    }

                    var topMapObject = (IEnumerable<KeyValuePair<string, JToken>>)parsedJson.Get("subscription");
                    if (topMapObject != null)
                    {
                        topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value<int>());
                    }
                    else
                    {
                        throw new KafkaException("error constructing TopicCount:" + topicCountString);
                    }
                }
                else
                {
                    throw new KafkaException("error constructing TopicCount:" + topicCountString);
                }
            }
            catch (Exception e)
            {
                Logger.Error("error parsing consumer json string " + topicCountString, e);
                throw;
            }

            var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern);
            var hasBlackList = BlackListPattern.Equals(subscriptionPattern);

            if (topMap.Count == 0 || !(hasWhiteList || hasBlackList))
            {
                return new StaticTopicCount(consumerId, topMap);
            }
            else
            {
                var regex = topMap.First().Key;
                var numStreams = topMap.First().Value;
                TopicFilter filter = hasWhiteList ? (TopicFilter)new Whitelist(regex) : new Blacklist(regex);
                return new WildcardTopicCount(zkClient, consumerId, filter, numStreams);
            }
        }
Esempio n. 20
0
 public void WaitForServersToSettle()
 {
     foreach (TempKafkaConfig config in this.Configs)
     {
         if (!ZkClient.WaitUntilExists(ZkUtils.BrokerIdsPath + "/" + config.BrokerId, TimeSpan.FromSeconds(5)))
         {
             throw new Exception("Timeout on waiting for broker " + config.BrokerId + " to settle");
         }
     }
 }
        public static void UpdateConsumerOffset(ConsumerConfig config, string path, long offset)
        {
            var zkClient = new ZkClient(
                config.ZooKeeper.ZkConnect,
                config.ZooKeeper.ZkSessionTimeoutMs,
                config.ZooKeeper.ZkConnectionTimeoutMs,
                new ZkStringSerializer());

            ZkUtils.UpdatePersistentPath(zkClient, path, offset.ToString(CultureInfo.InvariantCulture));
        }
        public static List <Broker> GetAllBrokersInCluster(ZkClient zkClient)
        {
            var brokerIds = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath).OrderBy(x => x).ToList();

            return
                (brokerIds.Select(int.Parse)
                 .Select(id => GetBrokerInfo(zkClient, id))
                 .Where(x => x != null)
                 .ToList());
        }
        public static int?GetLeaderForPartition(ZkClient zkClient, string topic, int partition)
        {
            var leaderAndIsrOpt = ReadDataMaybeNull(zkClient, GetTopicPartitionLeaderAndIsrPath(topic, partition)).Item1;

            if (leaderAndIsrOpt != null)
            {
                return(JObject.Parse(leaderAndIsrOpt).SelectToken("leader").Value <int>());
            }

            return(null);
        }
 public static IList <string> GetChildrenParentMayNotExist(ZkClient client, string path)
 {
     try
     {
         return(client.GetChildren(path));
     }
     catch (ZkNoNodeException)
     {
         return(null);
     }
 }
Esempio n. 25
0
 public static void CreateTopic(
     ZkClient zkClient,
     string topic,
     int partitions,
     int replicationFactor,
     Dictionary<string, string> topicConfig)
 {
     var brokerList = ZkUtils.GetSortedBrokerList(zkClient);
     var replicaAssigment = AssignReplicasToBrokers(brokerList, partitions, replicationFactor);
     CreateOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, replicaAssigment, topicConfig);
 }
 /// <summary>
 /// Create an ephemeral node with the given path and data. Create parents if necessary.
 /// </summary>
 /// <param name="client"></param>
 /// <param name="path"></param>
 /// <param name="data"></param>
 private static void CreateEphemeralPath(ZkClient client, string path, string data)
 {
     try
     {
         client.CreateEphemeral(path, data);
     }
     catch (ZkNoNodeException)
     {
         CreateParentPath(client, path);
         client.CreateEphemeral(path, data);
     }
 }
Esempio n. 27
0
        static void Main(string[] args)
        {
            ZkConnection zkConnection        = new ZkConnection("192.168.60.6:2181");
            ZkClient     zkClient            = new ZkClient(ZkConnect, ZkSessionTimeout, ZkConnectionTimeout, new ZkStringSerializer());
            var          dataLock            = new ReentrantLock();
            var          dataExistsOrChanged = dataLock.NewCondition();

            zkClient.SubscribeDataChanges("/testyy", new DataChange(dataLock, dataExistsOrChanged, zkClient));
            //dataExistsOrChanged.Await(TimeSpan.FromMilliseconds(1000));

            Console.Read();
        }
        public static void CreateTopic(
            ZkClient zkClient,
            string topic,
            int partitions,
            int replicationFactor,
            Dictionary <string, string> topicConfig)
        {
            var brokerList       = ZkUtils.GetSortedBrokerList(zkClient);
            var replicaAssigment = AssignReplicasToBrokers(brokerList, partitions, replicationFactor);

            CreateOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, replicaAssigment, topicConfig);
        }
Esempio n. 29
0
        public ConsumerFetcherManager(string consumerIdString, ConsumerConfig config, ZkClient zkClient)
            : base(
                string.Format("ConsumerFetcherManager-{0}", DateTimeHelper.CurrentTimeMilis()), config.ClientId, config.NumConsumerFetchers)
        {
            this.NoLeaderPartitionSet = new HashSet<TopicAndPartition>();
            this.consumerIdString = consumerIdString;
            this.config = config;
            this.zkClient = zkClient;

            this.@lock = new ReentrantLock();
            this.cond = [email protected]();
        }
 /// <summary>
 ///  Create an persistent node with the given path and data. Create parents if necessary.
 /// </summary>
 public static void CreatePersistentPath(ZkClient client, string path, string data)
 {
     try
     {
         client.CreatePersistent(path, data);
     }
     catch (ZkNoNodeException e)
     {
         CreateParentPath(client, path);
         client.CreatePersistent(path, data);
     }
 }
Esempio n. 31
0
        public ConsumerFetcherManager(string consumerIdString, ConsumerConfig config, ZkClient zkClient)
            : base(
                string.Format("ConsumerFetcherManager-{0}", DateTimeHelper.CurrentTimeMilis()), config.ClientId, config.NumConsumerFetchers)
        {
            this.NoLeaderPartitionSet = new HashSet <TopicAndPartition>();
            this.consumerIdString     = consumerIdString;
            this.config   = config;
            this.zkClient = zkClient;

            this.@lock = new ReentrantLock();
            this.cond  = [email protected]();
        }
 public static void MaybeDeletePath(string zkUrl, string dir)
 {
     try
     {
         var zk = new ZkClient(zkUrl, 30 * 1000, 30 * 1000, new ZkStringSerializer());
         zk.DeleteRecursive(dir);
         zk.Dispose();
     }
     catch
     {
         // swallow
     }
 }
 public static bool DeletePath(ZkClient client, string path)
 {
     try
     {
         return(client.Delete(path));
     }
     catch (ZkNoNodeException)
     {
         // this can happen during a connection loss event, return normally
         Logger.InfoFormat("{0} deleted during connection loss; This is ok. ", path);
         return(false);
     }
 }
        /// <summary>
        /// This API takes in a broker id, queries zookeeper for the broker metadata and returns the metadata for that broker
        /// or throws an exception if the broker dies before the query to zookeeper finishes
        /// </summary>
        /// <param name="zkClient">The zookeeper client connection</param>
        /// <param name="brokerId">The broker id</param>
        /// <returns>An optional Broker object encapsulating the broker metadata</returns>
        public static Broker GetBrokerInfo(ZkClient zkClient, int brokerId)
        {
            var brokerInfo = ReadDataMaybeNull(zkClient, BrokerIdsPath + "/" + brokerId);

            if (brokerInfo != null)
            {
                return(Broker.CreateBroker(brokerId, brokerInfo.Item1));
            }
            else
            {
                return(null);
            }
        }
        public static Cluster GetCluster(ZkClient zkClient)
        {
            var cluster = new Cluster();
            var nodes   = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath);

            foreach (var node in nodes)
            {
                var brokerZkString = ReadData(zkClient, BrokerIdsPath + "/" + node).Item1;
                cluster.Add(Broker.CreateBroker(int.Parse(node), brokerZkString));
            }

            return(cluster);
        }
        public static int?WaitUntilLeaderIsElectedOrChanged(
            ZkClient zkClient, string topic, int partition, long timeoutMs, int?oldLeaderOpt = null)
        {
            var leaderLock            = new ReentrantLock();
            var leaderExistsOrChanged = leaderLock.NewCondition();

            if (oldLeaderOpt.HasValue == false)
            {
                Logger.InfoFormat("Waiting for leader to be elected for partition [{0},{1}]", topic, partition);
            }
            else
            {
                Logger.InfoFormat("Waiting for leader for partition [{0},{1}] to be changed from old leader {2}", topic, partition, oldLeaderOpt.Value);
            }

            leaderLock.Lock();
            try
            {
                zkClient.SubscribeDataChanges(ZkUtils.GetTopicPartitionLeaderAndIsrPath(topic, partition), new LeaderExistsOrChangedListener(topic, partition, leaderLock, leaderExistsOrChanged, oldLeaderOpt, zkClient));
                leaderExistsOrChanged.Await(TimeSpan.FromMilliseconds(timeoutMs));

                // check if leader is elected
                var leader = ZkUtils.GetLeaderForPartition(zkClient, topic, partition);
                if (leader != null)
                {
                    if (oldLeaderOpt.HasValue == false)
                    {
                        Logger.InfoFormat("Leader {0} is elected for partition [{1},{2}]", leader, topic, partition);
                    }
                    else
                    {
                        Logger.InfoFormat(
                            "Leader for partition [{0},{1}] is changed from {2} to {3}",
                            topic,
                            partition,
                            oldLeaderOpt.Value,
                            leader);
                    }
                }
                else
                {
                    Logger.ErrorFormat("Timing out after {0} ms since leader is not elected for partition [{1},{2}]", timeoutMs, topic, partition);
                }

                return(leader);
            }
            finally
            {
                leaderLock.Unlock();
            }
        }
        public static Tuple <string, Stat> ReadDataMaybeNull(ZkClient client, string path)
        {
            var stat = new Stat();

            try
            {
                var obj = client.ReadData <string>(path, stat);
                return(Tuple.Create(obj, stat));
            }
            catch (ZkNoNodeException)
            {
                return(Tuple.Create((string)null, stat));
            }
        }
Esempio n. 38
0
        public void TestEphemeralNodeCleanup()
        {
            var zkClient = new ZkClient(ZkConnect, ZkSessionTimeoutMs, ZkConnectionTimeout, new ZkStringSerializer());

            try
            {
                ZkUtils.CreateEphemeralPathExpectConflict(zkClient, "/tmp/zktest", "node created");
            }
            catch
            {
                // ok
            }

            var testData = ZkUtils.ReadData(this.ZkClient, "/tmp/zktest").Item1;
            Assert.NotNull(testData);
            zkClient.Dispose();
            zkClient = new ZkClient(ZkConnect, ZkSessionTimeoutMs, ZkConnectionTimeout, new ZkStringSerializer());
            var nodeExists = ZkUtils.PathExists(zkClient, "/tmp/zktest");
            Assert.False(nodeExists);
        }
Esempio n. 39
0
         public static void CreateOrUpdateTopicPartitionAssignmentPathInZK(
             ZkClient zkClient,
             string topic,
             Dictionary<int, List<int>> partitionReplicaAssignment,
             Dictionary<string, string> config,
             bool update = false)
         {

             // Note: validation was omited
             var topicPath = ZkUtils.GetTopicPath(topic);
             if (!update && zkClient.Exists(topicPath))
             {
                 throw new KafkaException("Topic " + topic +" already exists.");
             }

             // write out the config if there is any, this isn't transactional with the partition assignments
             WriteTopicConfig(zkClient, topic, config);
    
            // create the partition assignment
             WriteTopicPartitionAssignment(zkClient, topic, partitionReplicaAssignment, update);
         }
 public ZookeeperTopicEventWatcher(ZkClient zkClient, ITopicEventHandler<string> eventHandler)
 {
     this.zkClient = zkClient;
     this.eventHandler = eventHandler;
     this.StartWatchingTopicEvents();
 }
Esempio n. 41
0
        /// <summary>
        /// Create an ephemeral node with the given path and data.
        /// Throw NodeExistsException if node already exists.
        /// Handles the following ZK session timeout b_u_g
        ///
        /// https://issues.apache.org/jira/browse/ZOOKEEPER-1740
        ///
        /// Upon receiving a NodeExistsException, read the data from the conflicted path and
        /// trigger the checker function comparing the read data and the expected data,
        /// If the checker function returns true then the above b_u_g might be encountered, back off and retry;
        /// otherwise re-throw the exception
        /// </summary>
        /// <param name="zkClient"></param>
        /// <param name="path"></param>
        /// <param name="data"></param>
        /// <param name="expectedCallerData"></param>
        /// <param name="checker"></param>
        /// <param name="backoffTime"></param>
        public static void CreateEphemeralPathExpectConflictHandleZKBug(
            ZkClient zkClient,
            string path,
            string data,
            object expectedCallerData,
            Func<string, object, bool> checker,
            int backoffTime)
        {
            while (true)
            {
                try
                {
                    CreateEphemeralPathExpectConflict(zkClient, path, data);
                    return;
                }
                catch (ZkNodeExistsException)
                {
                    // An ephemeral node may still exist even after its corresponding session has expired
                    // due to a Zookeeper ug, in this case we need to retry writing until the previous node is deleted
                    // and hence the write succeeds without ZkNodeExistsException
                    var writtenData = ReadDataMaybeNull(zkClient, path).Item1;
                    if (writtenData != null)
                    {
                        if (checker(writtenData, expectedCallerData))
                        {
                            Logger.InfoFormat(
                                "I wrote this conflicted ephemeral node [{0}] at {1} a while back in a different session, "
                                + "hence I will backoff for this node to be deleted by Zookeeper and retry",
                                data,
                                path);

                            Thread.Sleep(backoffTime);
                        }
                        else
                        {
                            throw;
                        }
                    }
                    else
                    {
                        // the node disappeared; retry creating the ephemeral node immediately
                    }
                }
            }
        }
Esempio n. 42
0
        public static int? GetLeaderForPartition(ZkClient zkClient, string topic, int partition)
        {
            var leaderAndIsrOpt = ReadDataMaybeNull(zkClient, GetTopicPartitionLeaderAndIsrPath(topic, partition)).Item1;
            if (leaderAndIsrOpt != null)
            {
                return JObject.Parse(leaderAndIsrOpt).SelectToken("leader").Value<int>();
            }

            return null;
        }
Esempio n. 43
0
 /// <summary>
 ///  Create an persistent node with the given path and data. Create parents if necessary.
 /// </summary>
 public static void CreatePersistentPath(ZkClient client, string path, string data)
 {
     try
     {
         client.CreatePersistent(path, data);
     }
     catch (ZkNoNodeException e)
     {
         CreateParentPath(client, path);
         client.CreatePersistent(path, data);
     }
 }
Esempio n. 44
0
 /// <summary>
 /// Update the value of a persistent node with the given path and data.
 ///  create parrent directory if necessary. Never throw NodeExistException.
 /// Return the updated path zkVersion
 /// </summary>
 /// <param name="client"></param>
 /// <param name="path"></param>
 /// <param name="data"></param>
 public static void UpdatePersistentPath(ZkClient client, string path, string data)
 {
     try
     {
         client.WriteData(path, data);
     }
     catch (ZkNoNodeException)
     {
         CreateParentPath(client, path);
         try
         {
             client.CreatePersistent(path, data);
         }
         catch (ZkNodeExistsException)
         {
             client.WriteData(path, data);
         }
     }
 }
Esempio n. 45
0
 public static Tuple<string, Stat> ReadDataMaybeNull(ZkClient client, string path)
 {
     var stat = new Stat();
     try
     {
         var obj = client.ReadData<string>(path, stat);
         return Tuple.Create(obj, stat);
     }
     catch (ZkNoNodeException)
     {
         return Tuple.Create((string)null, stat);
     }
 }
Esempio n. 46
0
        public static IDictionary<string, List<string>> GetConsumersPerTopic(ZkClient zkClient, string group)
        {
            var dirs = new ZKGroupDirs(group);
            var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir);
            var consumerPerTopicMap = new Dictionary<string, List<string>>();

            // Consumers are named by their group Id plus the machine name and an identifier suffix. In this context,
            // we are asked to retrieve the consumers for the `group` argument, so we discard any group in /consumers/ids
            // that is not prefixed with the group id (filter using Where Enumerable.Where extension method)
            foreach (var consumer in consumers.Where(c => c.StartsWith(group)))
            {
                var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient);
                foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic())
                {
                    var topic = topicAndConsumer.Key;
                    var consumerThreadIdSet = topicAndConsumer.Value;
                    foreach (var consumerThreadId in consumerThreadIdSet)
                    {
                        var curConsumers = consumerPerTopicMap.Get(topic);
                        if (curConsumers != null)
                        {
                            curConsumers.Add(consumerThreadId);
                        }
                        else
                        {
                            consumerPerTopicMap[topic] = new List<string> { consumerThreadId };
                        }
                    }
                }
            }

            consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList());

            return consumerPerTopicMap;
        }
Esempio n. 47
0
        public static IDictionary<string, IDictionary<int, List<int>>> GetPartitionAssignmentForTopics(
            ZkClient zkClient, IList<string> topics)
        {
            IDictionary<string, IDictionary<int, List<int>>> ret = new Dictionary<string, IDictionary<int, List<int>>>();
            foreach (var topic in topics)
            {
                var jsonPartitionMap = ReadDataMaybeNull(zkClient, GetTopicPath(topic)).Item1;
                IDictionary<int, List<int>> partitionMap = new Dictionary<int, List<int>>();
                if (jsonPartitionMap != null)
                {
                    var m = JObject.Parse(jsonPartitionMap);
                    var replicaMap = (IDictionary<string, JToken>)m.Get("partitions");
                    if (replicaMap != null)
                    {
                        partitionMap = replicaMap.ToDictionary(
                            kvp => int.Parse(kvp.Key), kvp => kvp.Value.Values<int>().ToList());
                    }
                }

                Logger.DebugFormat("Partition map for /brokers/topics/{0} is {1}", topic, JObject.FromObject(partitionMap).ToString(Formatting.None));
                ret[topic] = partitionMap;
            }

            return ret;
        }
Esempio n. 48
0
        public static Cluster GetCluster(ZkClient zkClient)
        {
            var cluster = new Cluster();
            var nodes = GetChildrenParentMayNotExist(zkClient, BrokerIdsPath);
            foreach (var node in nodes)
            {
                var brokerZkString = ReadData(zkClient, BrokerIdsPath + "/" + node).Item1;
                cluster.Add(Broker.CreateBroker(int.Parse(node), brokerZkString));
            }

            return cluster;
        }
Esempio n. 49
0
 public static bool PathExists(ZkClient client, string path)
 {
     return client.Exists(path);
 }
Esempio n. 50
0
 public static IList<string> GetChildrenParentMayNotExist(ZkClient client, string path)
 {
     try
     {
         return client.GetChildren(path);
     }
     catch (ZkNoNodeException)
     {
         return null;
     }
 }
Esempio n. 51
0
 public static List<string> GetChildren(ZkClient zkClient, string path)
 {
     return zkClient.GetChildren(path);
 } 
        public void TestLeaderSelectionForPartition()
        {
            var zkClient = new ZkClient(this.zookeeperConnect, 6000, 30000, new ZkStringSerializer());

            // create topic topic1 with 1 partition on broker 0
            AdminUtils.CreateTopic(zkClient, Topic, 1, 1, new Dictionary<string, string>());
            TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 3000);

            var sentMessages1 = this.SendMessages(
                Configs.First(), nMessages, "batch1", CompressionCodecs.NoCompressionCodec, 1);

            TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000);

            // create a consuemr
            var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1);
            var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1);
            var topicMessageStreams1 =
                zkConsumerConnector1.CreateMessageStreams(
                    new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder());

            var topicRegistry = zkConsumerConnector1.TopicRegistry;
            Assert.Equal(1, topicRegistry.Select(x => x.Key).Count());
            Assert.Equal(Topic, topicRegistry.Select(x => x.Key).First());

            var topicsAndPartitionsInRegistry =
                topicRegistry.Select(x => Tuple.Create(x.Key, x.Value.Select(p => p.Value))).ToList();

            var brokerPartition = topicsAndPartitionsInRegistry.First().Item2.First();
            Assert.Equal(0, brokerPartition.PartitionId);

            // also check partition ownership
            var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir);
            var expected_1 = new List<Tuple<string, string>>
                                 {
                                     Tuple.Create("0", "group1_consumer1-0"),
                                 };
            Assert.Equal(expected_1, actual_1);

            var receivedMessages1 = this.GetMessages(nMessages, topicMessageStreams1);
            Assert.Equal(sentMessages1, receivedMessages1);
            zkConsumerConnector1.Shutdown();
            zkClient.Dispose();
        }
Esempio n. 53
0
 public WildcardTopicCount(ZkClient zkClient, string consumerIdString, TopicFilter topicFilter, int numStreams)
 {
     this.ZkClient = zkClient;
     this.ConsumerIdString = consumerIdString;
     this.TopicFilter = topicFilter;
     this.NumStreams = numStreams;
 }
Esempio n. 54
0
 public static WildcardTopicCount ConstructTopicCount(
     string consumerIdString, TopicFilter filter, int numStream, ZkClient zkClient)
 {
     return new WildcardTopicCount(zkClient, consumerIdString, filter, numStream);
 }
Esempio n. 55
0
 /// <summary>
 /// This API takes in a broker id, queries zookeeper for the broker metadata and returns the metadata for that broker
 /// or throws an exception if the broker dies before the query to zookeeper finishes
 /// </summary>
 /// <param name="zkClient">The zookeeper client connection</param>
 /// <param name="brokerId">The broker id</param>
 /// <returns>An optional Broker object encapsulating the broker metadata</returns>
 public static Broker GetBrokerInfo(ZkClient zkClient, int brokerId)
 {
     var brokerInfo = ReadDataMaybeNull(zkClient, BrokerIdsPath + "/" + brokerId);
     if (brokerInfo != null)
     {
         return Broker.CreateBroker(brokerId, brokerInfo.Item1);
     }
     else
     {
         return null;
     }
 }
Esempio n. 56
0
 public static Tuple<string, Stat> ReadData(ZkClient client, string path)
 {
      var stat = new Stat();
      var dataString = client.ReadData<string>(path, stat);
      return Tuple.Create(dataString, stat);
 }
Esempio n. 57
0
 public LeaderExistsOrChangedListener(string topic, int partition, ReentrantLock leaderLock, ICondition leaderExistsOrChanged, int? oldLeaderOpt, ZkClient zkClient)
 {
     this.topic = topic;
     this.partition = partition;
     this.leaderLock = leaderLock;
     this.leaderExistsOrChanged = leaderExistsOrChanged;
     this.oldLeaderOpt = oldLeaderOpt;
     this.zkClient = zkClient;
 }
Esempio n. 58
0
 public static List<int> GetSortedBrokerList(ZkClient zkClient)
 {
     return GetChildren(zkClient, BrokerIdsPath).Select(int.Parse).OrderBy(x => x).ToList();
 } 
Esempio n. 59
0
 public static void MaybeDeletePath(string zkUrl, string dir)
 {
     try
     {
         var zk = new ZkClient(zkUrl, 30 * 1000, 30 * 1000, new ZkStringSerializer());
         zk.DeleteRecursive(dir);
         zk.Dispose();
     }
     catch
     {
         // swallow
     }
 }
Esempio n. 60
0
 public static bool DeletePath(ZkClient client, string path)
 {
     try
     {
         return client.Delete(path);
     }
     catch (ZkNoNodeException)
     {
         // this can happen during a connection loss event, return normally
         Logger.InfoFormat("{0} deleted during connection loss; This is ok. ", path);
         return false;
     }
 }