public void ConsumerPorformsRebalancingOnStart() { var config = this.ZooKeeperBasedConsumerConfig; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField <ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary <string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); IList <string> children = client.GetChildren("/consumers", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("group1")); children = client.GetChildren("/consumers/group1", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("ids")); Assert.That(children, Contains.Item("owners")); children = client.GetChildren("/consumers/group1/ids", false); Assert.That(children, Is.Not.Null.And.Not.Empty); string consumerId = children[0]; children = client.GetChildren("/consumers/group1/owners", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(1)); Assert.That(children, Contains.Item("test")); children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(2)); string partId = children[0]; var data = client.ReadData <string>("/consumers/group1/owners/test/" + partId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Contains.Substring(consumerId)); data = client.ReadData <string>("/consumers/group1/ids/" + consumerId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Is.EqualTo("{ \"test\": 1 }")); } using (var client = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); //// Should be created as ephemeral IList <string> children = client.GetChildren("/consumers/group1/ids"); Assert.That(children, Is.Null.Or.Empty); //// Should be created as ephemeral children = client.GetChildren("/consumers/group1/owners/test"); Assert.That(children, Is.Null.Or.Empty); } }
private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List <ConsumeGroupMonitorUnit> units) { Logger.Info("Will refresh all consume group from zk ..."); string path = "/consumers"; //TODO: add /users IEnumerable <string> groups = zkClient.GetChildren(path); foreach (var g in groups) { Logger.InfoFormat("Will check group {0}", g); string topicpath = string.Format("/consumers/{0}/owners", g); IEnumerable <string> topics = zkClient.GetChildren(topicpath); foreach (var t in topics) { Logger.InfoFormat("Will check topic {0}", t); units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t)); } } Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count); }
public void ZooKeeperClientCreatesAChildAndGetsChildren() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); string child = Guid.NewGuid().ToString(); string myPath = "/" + child; client.CreatePersistent(myPath, false); IList <string> children = client.GetChildren("/", false); int countChildren = client.CountChildren("/"); Assert.Greater(children.Count, 0); Assert.AreEqual(children.Count, countChildren); Assert.IsTrue(children.Contains(child)); client.Delete(myPath); } }
protected override List <string> GetGroupPaths(string rootPath) { return(_zkClient.GetChildren(rootPath, false).ToList()); }
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions) { string topics = dtOptions.Topic; string zookeeper = dtOptions.Zookeeper; int partitionIndex = dtOptions.PartitionIndex; bool includePartitionDetailInfo = true; bool includeOffsetInfo = true; DateTime timestamp = dtOptions.TimestampInUTC; bool dumpToLog = true; bool dumpToConsole = true; string file = dtOptions.File; using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); //BrokerID -->Count of as leader SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>(); //BrokerID -->Count of as replica SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>(); StringBuilder sbAll = new StringBuilder(); int topicCount = 0; if (string.IsNullOrEmpty(topics)) { List <string> topicList = new List <string>(); string path = "/brokers/topics"; try { IEnumerable <string> ts = zkClient.GetChildren(path); foreach (var p in ts) { topicList.Add(p); } } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e); } else { throw; } } sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count); foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray()) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } topicCount = topicList.Count; } else if (topics.Contains(",")) { string[] topicArray = topics.Split(new char[] { ',' }); topicCount = topicArray.Length; sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length); foreach (var t in topicArray.OrderBy(r => r).ToArray()) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } } else { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); topicCount = 1; } if (topicCount > 1) { sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n"); sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n"); foreach (KeyValuePair <int, int> kv in parttionBrokerID_LeaderCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n"); sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n"); foreach (KeyValuePair <int, int> kv in parttionBrokerID_ReplicaCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } } string s = sbAll.ToString(); if (dumpToLog) { Logger.Info(s); } if (dumpToConsole) { Console.WriteLine(s); } if (!string.IsNullOrEmpty(file)) { Console.WriteLine("Will write to {0}", file); using (StreamWriter sw = new StreamWriter(file, false)) { sw.WriteLine(s); } } } }
public void GetChildren_should_not_add_watch_triggered_by_Created() { var path = "/watch/new"; var watcher = new TestWatcher(); client.GetChildren(new GetChildrenRequest(path) { Watcher = watcher }).Status.Should().Be(ZooKeeperStatus.NodeNotFound); client.Create(new CreateRequest(path, CreateMode.Persistent)).EnsureSuccess(); watcher.ShouldNotBeTriggered(); }