private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List<ConsumeGroupMonitorUnit> units)
{
Logger.Info("Will refresh all consume group from zk ...");
string path = "/consumers";
//TODO: add /users
IEnumerable<string> groups = zkClient.GetChildren(path);
foreach (var g in groups)
{
Logger.InfoFormat("Will check group {0}", g);
string topicpath = string.Format("/consumers/{0}/owners", g);
IEnumerable<string> topics = zkClient.GetChildren(topicpath);
foreach (var t in topics)
{
Logger.InfoFormat("Will check topic {0}", t);
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t));
}
}
Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count);
}
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions)
{
string topics = dtOptions.Topic;
string zookeeper = dtOptions.Zookeeper;
int partitionIndex = dtOptions.PartitionIndex;
bool includePartitionDetailInfo = true;
bool includeOffsetInfo = true;
DateTime timestamp = dtOptions.TimestampInUTC;
bool dumpToLog = true;
bool dumpToConsole = true;
string file = dtOptions.File;
using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
//BrokerID -->Count of as leader
SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
//BrokerID -->Count of as replica
SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
StringBuilder sbAll = new StringBuilder();
int topicCount = 0;
if (string.IsNullOrEmpty(topics))
{
List<string> topicList = new List<string>();
string path = "/brokers/topics";
try
{
IEnumerable<string> ts = zkClient.GetChildren(path);
foreach (var p in ts)
{
topicList.Add(p);
}
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e);
}
else
throw;
}
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count);
foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray())
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
topicCount = topicList.Count;
}
else if (topics.Contains(","))
{
string[] topicArray = topics.Split(new char[] { ',' });
topicCount = topicArray.Length;
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length);
foreach (var t in topicArray.OrderBy(r => r).ToArray())
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
}
else
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
topicCount = 1;
}
if (topicCount > 1)
{
sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n");
sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_LeaderCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n");
sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_ReplicaCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
}
string s = sbAll.ToString();
if (dumpToLog)
{
Logger.Info(s);
}
if (dumpToConsole)
{
Console.WriteLine(s);
}
if (!string.IsNullOrEmpty(file))
{
Console.WriteLine("Will write to {0}", file);
using (StreamWriter sw = new StreamWriter(file, false))
{
sw.WriteLine(s);
}
}
}
}
public void ZooKeeperClientCreatesAChildAndGetsChildren()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
string child = Guid.NewGuid().ToString();
string myPath = "/" + child;
client.CreatePersistent(myPath, false);
IList<string> children = client.GetChildren("/", false);
int countChildren = client.CountChildren("/");
Assert.Greater(children.Count, 0);
Assert.AreEqual(children.Count, countChildren);
Assert.IsTrue(children.Contains(child));
client.Delete(myPath);
}
}
public void ConsumerPorformsRebalancingOnStart()
{
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1" };
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
ZooKeeperClient client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
IList<string> children = client.GetChildren("/consumers", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("group1"));
children = client.GetChildren("/consumers/group1", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("ids"));
Assert.That(children, Contains.Item("owners"));
children = client.GetChildren("/consumers/group1/ids", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
string consumerId = children[0];
children = client.GetChildren("/consumers/group1/owners", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(1));
Assert.That(children, Contains.Item("test"));
children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(2));
string partId = children[0];
string data = client.ReadData<string>("/consumers/group1/owners/test/" + partId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Contains.Substring(consumerId));
data = client.ReadData<string>("/consumers/group1/ids/" + consumerId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Is.EqualTo("{ \"test\": 1 }"));
}
using (var client = new ZooKeeperClient(config.ZkConnect, config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
{
client.Connect();
//// Should be created as ephemeral
IList<string> children = client.GetChildren("/consumers/group1/ids");
Assert.That(children, Is.Null.Or.Empty);
//// Should be created as ephemeral
children = client.GetChildren("/consumers/group1/owners/test");
Assert.That(children, Is.Null.Or.Empty);
}
}
