public void WhenStateChangedToExpiredStateAndSessionListenersFire()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Subscribe(this);
client.Connect();
WaitUntillIdle(client, 500);
client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
WaitUntillIdle(client, 3000);
}
Assert.AreEqual(4, this.events.Count);
ZooKeeperEventArgs e = this.events[1];
Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
Assert.IsInstanceOf <ZooKeeperStateChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.Expired);
e = this.events[2];
Assert.AreEqual(ZooKeeperEventTypes.SessionCreated, e.Type);
Assert.IsInstanceOf <ZooKeeperSessionCreatedEventArgs>(e);
e = this.events[3];
Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
Assert.IsInstanceOf <ZooKeeperStateChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
}
public void WhenChildIsDeletedChildListenerOnParentFires()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
string myPath = "/" + Guid.NewGuid();
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
client.CreatePersistent(myPath, true);
WaitUntillIdle(client, 500);
client.Subscribe("/", this as IZooKeeperChildListener);
client.Delete(myPath);
WaitUntillIdle(client, 500);
}
Assert.AreEqual(1, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
Assert.IsInstanceOf <ZooKeeperChildChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, "/");
Assert.Greater(((ZooKeeperChildChangedEventArgs)e).Children.Count, 0);
Assert.IsFalse(((ZooKeeperChildChangedEventArgs)e).Children.Contains(myPath.Replace("/", string.Empty)));
}
public void WhenZNodeIsDeletedChildAndDataDeletedListenersFire()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
string myPath = "/" + Guid.NewGuid();
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
client.CreatePersistent(myPath, true);
WaitUntillIdle(client, 500);
client.Subscribe(myPath, this as IZooKeeperChildListener);
client.Subscribe(myPath, this as IZooKeeperDataListener);
client.Delete(myPath);
WaitUntillIdle(client, 500);
}
Assert.AreEqual(2, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
Assert.IsInstanceOf <ZooKeeperChildChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, myPath);
Assert.IsNull(((ZooKeeperChildChangedEventArgs)e).Children);
e = this.events[1];
Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
Assert.IsInstanceOf <ZooKeeperDataChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
Assert.IsNull(((ZooKeeperDataChangedEventArgs)e).Data);
}
internal static void Run(ProduceSimpleHelperOption produceOptions)
{
failedMessageCount = 0;
successMessageCount = 0;
sentBatchCount = 0;
produceMessagePerPartition = new Dictionary <int, int>();
produceMessagePerPartitionExpect = new Dictionary <int, int>();
PrepareSentMessages(produceOptions);
kafkaSimpleManagerConfig = new KafkaSimpleManagerConfiguration()
{
Zookeeper = produceOptions.Zookeeper,
MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize,
PartitionerClass = produceOptions.PartitionerClass
};
kafkaSimpleManagerConfig.Verify();
producerConfigTemplate = new ProducerConfiguration(
new List <BrokerConfiguration>()
{
}) //The Brokers will be replaced inside of KafkaSimpleManager
{
ForceToPartition = -1,
PartitionerClass = kafkaSimpleManagerConfig.PartitionerClass,
TotalNumPartitions = 0,
RequiredAcks = produceOptions.RequiredAcks,
AckTimeout = produceOptions.AckTimeout,
SendTimeout = produceOptions.SendTimeout,
ReceiveTimeout = produceOptions.ReceiveTimeout,
CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()),
BufferSize = produceOptions.BufferSize,
SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition
MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize)
};
using (ZooKeeperClient zkClient = new ZooKeeperClient(produceOptions.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
Dictionary <int, int[]> topicDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, produceOptions.Topic);
// -2 by default or customized partitioner class. (you need specify PartitionerClass by -l)
if (produceOptions.PartitionId == -2)
{
if (string.IsNullOrEmpty(kafkaSimpleManagerConfig.PartitionerClass))
{
throw new ArgumentException("The partitioer class must not be empty if you want to send to partition by partitioner.");
}
//if (producerConfigTemplate.TotalNumPartitions <= 0)
// throw new ArgumentException(string.Format("Please also specify the TotalNumPartitions if you want to send to partition by partitioner."));
ProduceByPartitionerClass(produceOptions, topicDataInZookeeper.Count);
}
else
{
ProduceToRandomOrSpecificPartition(produceOptions);
}
}
}
public ZookeeperClient(string zkConnect)
{
_client = new ZooKeeperClient(
zkConnect,
ZooKeeperConfiguration.DefaultSessionTimeout,
ZooKeeperStringSerializer.Serializer
);
_client.Connect();
}
public void WhenNewBrokerInTopicIsAddedBrokerTopicsListenerUpdatesMappings()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
IDictionary <string, SortedSet <Partition> > mappings;
IDictionary <int, Broker> brokers;
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic;
string topicBrokerPath = topicPath + "/" + 2345;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
{
brokers = brokerPartitionInfo.GetAllBrokerInfo();
mappings =
ReflectionHelper.GetInstanceField <IDictionary <string, SortedSet <Partition> > >(
"topicBrokerPartitions", brokerPartitionInfo);
}
}
Assert.NotNull(brokers);
Assert.Greater(brokers.Count, 0);
Assert.NotNull(mappings);
Assert.Greater(mappings.Count, 0);
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
WaitUntillIdle(client, 500);
var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null);
client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener);
client.Subscribe(ZooKeeperClient.DefaultBrokerTopicsPath, brokerTopicsListener);
client.CreatePersistent(brokerPath, true);
client.WriteData(brokerPath, "172.16.66.2-1310449279123:172.16.66.2:9092");
client.CreatePersistent(topicPath, true);
WaitUntillIdle(client, 500);
Assert.IsTrue(brokers.ContainsKey(2345));
Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic));
client.CreatePersistent(topicBrokerPath, true);
client.WriteData(topicBrokerPath, 5);
WaitUntillIdle(client, 500);
client.UnsubscribeAll();
WaitUntillIdle(client, 500);
client.DeleteRecursive(brokerPath);
client.DeleteRecursive(topicPath);
}
Assert.IsTrue(brokers.ContainsKey(2345));
Assert.IsTrue(mappings.Keys.Contains(CurrentTestTopic));
Assert.AreEqual(5, mappings[CurrentTestTopic].Count);
}
public void ZooKeeperClientChecksIfPathExists()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
Assert.IsTrue(client.Exists(ZooKeeperClient.DefaultBrokerTopicsPath, false));
}
}
public void ConsumerPorformsRebalancingOnStart()
{
var config = this.ZooKeeperBasedConsumerConfig;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField <ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary <string, int> {
{ "test", 1 }
};
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
IList <string> children = client.GetChildren("/consumers", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("group1"));
children = client.GetChildren("/consumers/group1", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("ids"));
Assert.That(children, Contains.Item("owners"));
children = client.GetChildren("/consumers/group1/ids", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
string consumerId = children[0];
children = client.GetChildren("/consumers/group1/owners", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(1));
Assert.That(children, Contains.Item("test"));
children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(2));
string partId = children[0];
var data = client.ReadData <string>("/consumers/group1/owners/test/" + partId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Contains.Substring(consumerId));
data = client.ReadData <string>("/consumers/group1/ids/" + consumerId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Is.EqualTo("{ \"test\": 1 }"));
}
using (var client = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
{
client.Connect();
//// Should be created as ephemeral
IList <string> children = client.GetChildren("/consumers/group1/ids");
Assert.That(children, Is.Null.Or.Empty);
//// Should be created as ephemeral
children = client.GetChildren("/consumers/group1/owners/test");
Assert.That(children, Is.Null.Or.Empty);
}
}
public SyncProducerPool(ProducerConfiguration config)
{
syncProducers = new ConcurrentDictionary <int, SyncProducerWrapper>();
Config = config;
if (config.ZooKeeper != null)
{
zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer);
zkClient.Connect();
}
AddProducers(config);
}
public void WhenClientWillNotConnectWithinGivenTimeThrows()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client =
new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer,
1))
{
client.Connect();
}
}
public void ZooKeeperClientCreateWorkerThreadsOnBeingCreated()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
var eventWorker = ReflectionHelper.GetInstanceField <Thread>("eventWorker", client);
var zooKeeperWorker = ReflectionHelper.GetInstanceField <Thread>("zooKeeperEventWorker", client);
Assert.NotNull(eventWorker);
Assert.NotNull(zooKeeperWorker);
}
}
public void ZooKeeperClientCreatesANewPathAndDeletesIt()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
string myPath = "/" + Guid.NewGuid();
client.CreatePersistent(myPath, false);
Assert.IsTrue(client.Exists(myPath));
client.Delete(myPath);
Assert.IsFalse(client.Exists(myPath));
}
}
public SyncProducerPool(ProducerConfiguration config, List <ISyncProducer> producers)
{
syncProducers = new ConcurrentDictionary <int, SyncProducerWrapper>();
Config = config;
if (config.ZooKeeper != null)
{
zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer);
zkClient.Connect();
}
if (producers != null && producers.Any())
{
producers.ForEach(x => syncProducers.TryAdd(x.Config.BrokerId,
new SyncProducerWrapper(x, config.SyncProducerOfOneBroker)));
}
}
public void WhenBrokerIsRemovedBrokerTopicsListenerUpdatesBrokersList()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
IDictionary <string, SortedSet <Partition> > mappings;
IDictionary <int, Broker> brokers;
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
{
brokers = brokerPartitionInfo.GetAllBrokerInfo();
mappings =
ReflectionHelper.GetInstanceField <IDictionary <string, SortedSet <Partition> > >(
"topicBrokerPartitions", brokerPartitionInfo);
}
}
Assert.NotNull(brokers);
Assert.Greater(brokers.Count, 0);
Assert.NotNull(mappings);
Assert.Greater(mappings.Count, 0);
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
WaitUntillIdle(client, 500);
var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null);
client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener);
client.CreatePersistent(brokerPath, true);
client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
WaitUntillIdle(client, 500);
Assert.IsTrue(brokers.ContainsKey(2345));
client.DeleteRecursive(brokerPath);
WaitUntillIdle(client, 500);
Assert.IsFalse(brokers.ContainsKey(2345));
}
}
public void WhenStateChangedToConnectedStateListenerFires()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Subscribe(this);
client.Connect();
WaitUntillIdle(client, 500);
}
Assert.AreEqual(1, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
Assert.IsInstanceOf <ZooKeeperStateChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
}
public void ZooKeeperClientCreatesAChildAndGetsChildren()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
string child = Guid.NewGuid().ToString();
string myPath = "/" + child;
client.CreatePersistent(myPath, false);
IList <string> children = client.GetChildren("/", false);
int countChildren = client.CountChildren("/");
Assert.Greater(children.Count, 0);
Assert.AreEqual(children.Count, countChildren);
Assert.IsTrue(children.Contains(child));
client.Delete(myPath);
}
}
public void WhenSessionExpiredClientReconnects()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
IZooKeeperConnection conn1;
IZooKeeperConnection conn2;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
conn1 = ReflectionHelper.GetInstanceField <ZooKeeperConnection>("connection", client);
client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
WaitUntillIdle(client, 1000);
conn2 = ReflectionHelper.GetInstanceField <ZooKeeperConnection>("connection", client);
}
Assert.AreNotEqual(conn1, conn2);
}
public void WhenDataChangedDataListenerFires()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
string myPath = "/" + Guid.NewGuid();
string sourceData = "my test data";
string resultData;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
client.CreatePersistent(myPath, true);
WaitUntillIdle(client, 500);
client.Subscribe(myPath, this as IZooKeeperDataListener);
client.Subscribe(myPath, this as IZooKeeperChildListener);
client.WriteData(myPath, sourceData);
WaitUntillIdle(client, 500);
client.UnsubscribeAll();
resultData = client.ReadData <string>(myPath);
client.Delete(myPath);
}
Assert.IsTrue(!string.IsNullOrEmpty(resultData));
Assert.AreEqual(sourceData, resultData);
Assert.AreEqual(1, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
Assert.IsInstanceOf <ZooKeeperDataChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
Assert.IsNotNull(((ZooKeeperDataChangedEventArgs)e).Data);
Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Data, sourceData);
}
internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption)
{
List <ConsumeGroupMonitorUnit> units = new List <ConsumeGroupMonitorUnit>();
if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray))
{
if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName))
{
units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic));
}
else
{
danymicAllConsumeGroupTopic = true;
}
}
else
{
string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var cg in cgs)
{
string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
if (temp.Length != 2)
{
Logger.ErrorFormat("Wrong parameter, Exmaple G1:t1,G2:t2. The value:{0} is invalid.", cg);
}
else
{
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1]));
}
}
}
long count = 0;
int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds;
using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
while (true)
{
DateTime time = DateTime.Now;
string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day);
if (!Directory.Exists(dateFolder))
{
Directory.CreateDirectory(dateFolder);
}
if (danymicAllConsumeGroupTopic && count % cycle == 0)
{
units = new List <ConsumeGroupMonitorUnit>();
RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units);
}
int countSuccess = 0;
foreach (var unit in units)
{
unit.subFolder = dateFolder;
if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper))
{
countSuccess++;
}
}
if (countSuccess == units.Count)
{
Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess);
}
else
{
Logger.ErrorFormat("===========All {0} consume group only {1} succ. FAIL=================", units.Count, countSuccess);
}
Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000);
count++;
}
}
}
internal static void Run(ProduceMonitorHelperOptions produceMonitorOptions)
{
using (ZooKeeperClient zkClient = new ZooKeeperClient(produceMonitorOptions.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
while (true)
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>();
//BrokerID -->Count of as replica
SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, produceMonitorOptions.Topic,
produceMonitorOptions.Zookeeper,
-1,
true,
true,
DateTime.MinValue,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength);
if (latestOffsetDictLastValue == null)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("====Partitions====\r\n");
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Key);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====LatestOffset====\r\n");
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
}
else
{
StringBuilder sb = new StringBuilder();
sb.Append("Latest Delta: ");
foreach (KeyValuePair <int, long> kv in latestOffsetDictLastValue)
{
if (latestOffsetDict.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value);
}
else
{
sb.AppendFormat("Latest:{0,-9} ", kv.Value);
}
}
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
if (!latestOffsetDictLastValue.ContainsKey(kv.Key))
{
sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value);
}
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
}
latestOffsetDictLastValue = latestOffsetDict;
Thread.Sleep(produceMonitorOptions.IntervalInSeconds * 1000);
}
}
}
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions)
{
string topics = dtOptions.Topic;
string zookeeper = dtOptions.Zookeeper;
int partitionIndex = dtOptions.PartitionIndex;
bool includePartitionDetailInfo = true;
bool includeOffsetInfo = true;
DateTime timestamp = dtOptions.TimestampInUTC;
bool dumpToLog = true;
bool dumpToConsole = true;
string file = dtOptions.File;
using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
//BrokerID -->Count of as leader
SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>();
//BrokerID -->Count of as replica
SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>();
StringBuilder sbAll = new StringBuilder();
int topicCount = 0;
if (string.IsNullOrEmpty(topics))
{
List <string> topicList = new List <string>();
string path = "/brokers/topics";
try
{
IEnumerable <string> ts = zkClient.GetChildren(path);
foreach (var p in ts)
{
topicList.Add(p);
}
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e);
}
else
{
throw;
}
}
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count);
foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray())
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
topicCount = topicList.Count;
}
else if (topics.Contains(","))
{
string[] topicArray = topics.Split(new char[] { ',' });
topicCount = topicArray.Length;
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length);
foreach (var t in topicArray.OrderBy(r => r).ToArray())
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
}
else
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
topicCount = 1;
}
if (topicCount > 1)
{
sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n");
sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
foreach (KeyValuePair <int, int> kv in parttionBrokerID_LeaderCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n");
sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
foreach (KeyValuePair <int, int> kv in parttionBrokerID_ReplicaCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
}
string s = sbAll.ToString();
if (dumpToLog)
{
Logger.Info(s);
}
if (dumpToConsole)
{
Console.WriteLine(s);
}
if (!string.IsNullOrEmpty(file))
{
Console.WriteLine("Will write to {0}", file);
using (StreamWriter sw = new StreamWriter(file, false))
{
sw.WriteLine(s);
}
}
}
}
