public BrokerPartitionInfo(ISyncProducerPool syncProducerPool, IDictionary<string, TopicMetadata> cache, IDictionary<string, DateTime> lastUpdateTime, int topicMetaDataRefreshIntervalMS, ZooKeeperClient zkClient)
{
this.syncProducerPool = syncProducerPool;
this.topicPartitionInfo = cache;
this.topicPartitionInfoLastUpdateTime = lastUpdateTime;
this.topicMetaDataRefreshIntervalMS = topicMetaDataRefreshIntervalMS;
this.zkClient = zkClient;
}
public SyncProducerPool(ProducerConfiguration config)
{
this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>();
this.Config = config;
if (config.ZooKeeper != null)
{
this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer);
this.zkClient.Connect();
}
this.AddProducers(config);
}
public SyncProducerPool(ProducerConfiguration config, List<ISyncProducer> producers)
{
this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>();
this.Config = config;
if (config.ZooKeeper != null)
{
this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer);
this.zkClient.Connect();
}
if (producers != null && producers.Any())
{
producers.ForEach(x => this.syncProducers.TryAdd(x.Config.BrokerId, new SyncProducerWrapper(x, config.SyncProducerOfOneBroker)));
}
}
public void WhenBrokerIsRemovedBrokerTopicsListenerUpdatesBrokersList()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
IDictionary<string, SortedSet<Partition>> mappings;
IDictionary<int, Broker> brokers;
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
{
brokers = brokerPartitionInfo.GetAllBrokerInfo();
mappings =
ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
"topicBrokerPartitions", brokerPartitionInfo);
}
}
Assert.NotNull(brokers);
Assert.Greater(brokers.Count, 0);
Assert.NotNull(mappings);
Assert.Greater(mappings.Count, 0);
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
WaitUntillIdle(client, 500);
var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null);
client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener);
client.CreatePersistent(brokerPath, true);
client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
WaitUntillIdle(client, 500);
Assert.IsTrue(brokers.ContainsKey(2345));
client.DeleteRecursive(brokerPath);
WaitUntillIdle(client, 500);
Assert.IsFalse(brokers.ContainsKey(2345));
}
}
internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption)
{
List<ConsumeGroupMonitorUnit> units = new List<ConsumeGroupMonitorUnit>();
if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray))
{
if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName))
units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic));
else
{
danymicAllConsumeGroupTopic = true;
}
}
else
{
string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var cg in cgs)
{
string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
if (temp.Length != 2)
{
Logger.ErrorFormat("Wrong parameter, Exmaple G1:t1,G2:t2. The value:{0} is invalid.", cg);
}
else
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1]));
}
}
long count = 0;
int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds;
using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
while (true)
{
DateTime time = DateTime.Now;
string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day);
if (!Directory.Exists(dateFolder))
Directory.CreateDirectory(dateFolder);
if (danymicAllConsumeGroupTopic && count % cycle == 0)
{
units = new List<ConsumeGroupMonitorUnit>();
RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units);
}
int countSuccess = 0;
foreach (var unit in units)
{
unit.subFolder = dateFolder;
if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper))
countSuccess++;
}
if (countSuccess == units.Count)
Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess);
else
Logger.ErrorFormat("===========All {0} consume group only {1} succ. FAIL=================", units.Count, countSuccess);
Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000);
count++;
}
}
}
internal static SortedDictionary<int, string> GetComsumerGroupOwners(ZooKeeperClient zkClient,
string topic, string consumerGroupName)
{
SortedDictionary<int, string> partitionsOwners = new SortedDictionary<int, string>();
string path = string.Format("/consumers/{0}/owners/{1}"
, consumerGroupName, topic);
IEnumerable<string> partitions = zkClient.GetChildrenParentMayNotExist(path);
if (partitions != null)
{
foreach (var p in partitions)
{
string fullPatht = string.Format("/consumers/{0}/owners/{1}/{2}"
, consumerGroupName, topic, p);
string data = zkClient.ReadData<string>(fullPatht, true);
partitionsOwners.Add(Convert.ToInt32(p), data);
}
}
return partitionsOwners;
}
public void WhenChildIsCreatedChilListenerOnParentFires()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
string myPath = "/" + Guid.NewGuid();
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
WaitUntillIdle(client, 500);
client.Subscribe("/", this as IZooKeeperChildListener);
client.CreatePersistent(myPath, true);
WaitUntillIdle(client, 500);
client.UnsubscribeAll();
client.Delete(myPath);
}
Assert.AreEqual(1, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperChildChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, "/");
Assert.Greater(((ZooKeeperChildChangedEventArgs)e).Children.Count, 0);
Assert.IsTrue(((ZooKeeperChildChangedEventArgs)e).Children.Contains(myPath.Replace("/", string.Empty)));
}
public void ZooKeeperClientCreateWorkerThreadsOnBeingCreated()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
var eventWorker = ReflectionHelper.GetInstanceField<Thread>("eventWorker", client);
var zooKeeperWorker = ReflectionHelper.GetInstanceField<Thread>("zooKeeperEventWorker", client);
Assert.NotNull(eventWorker);
Assert.NotNull(zooKeeperWorker);
}
}
public void ZooKeeperClientCreatesAChildAndGetsChildren()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
string child = Guid.NewGuid().ToString();
string myPath = "/" + child;
client.CreatePersistent(myPath, false);
IList<string> children = client.GetChildren("/", false);
int countChildren = client.CountChildren("/");
Assert.Greater(children.Count, 0);
Assert.AreEqual(children.Count, countChildren);
Assert.IsTrue(children.Contains(child));
client.Delete(myPath);
}
}
public void WhenZNodeIsDeletedChildAndDataDeletedListenersFire()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
string myPath = "/" + Guid.NewGuid();
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
client.CreatePersistent(myPath, true);
WaitUntillIdle(client, 500);
client.Subscribe(myPath, this as IZooKeeperChildListener);
client.Subscribe(myPath, this as IZooKeeperDataListener);
client.Delete(myPath);
WaitUntillIdle(client, 500);
}
Assert.AreEqual(2, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperChildChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, myPath);
Assert.IsNull(((ZooKeeperChildChangedEventArgs)e).Children);
e = this.events[1];
Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperDataChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
Assert.IsNull(((ZooKeeperDataChangedEventArgs)e).Data);
}
public void WhenStateChangedToDisconnectedStateListenerFires()
{
var producerConfig = new ProducerConfig(clientConfig);
using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
{
client.Subscribe(this);
client.Connect();
WaitUntillIdle(client, 500);
client.Process(new WatchedEvent(KeeperState.Disconnected, EventType.None, null));
WaitUntillIdle(client, 500);
}
Assert.AreEqual(2, this.events.Count);
ZooKeeperEventArgs e = this.events[1];
Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.Disconnected);
}
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions)
{
string topics = dtOptions.Topic;
string zookeeper = dtOptions.Zookeeper;
int partitionIndex = dtOptions.PartitionIndex;
bool includePartitionDetailInfo = true;
bool includeOffsetInfo = true;
DateTime timestamp = dtOptions.TimestampInUTC;
bool dumpToLog = true;
bool dumpToConsole = true;
string file = dtOptions.File;
using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
//BrokerID -->Count of as leader
SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
//BrokerID -->Count of as replica
SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
StringBuilder sbAll = new StringBuilder();
int topicCount = 0;
if (string.IsNullOrEmpty(topics))
{
List<string> topicList = new List<string>();
string path = "/brokers/topics";
try
{
IEnumerable<string> ts = zkClient.GetChildren(path);
foreach (var p in ts)
{
topicList.Add(p);
}
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e);
}
else
throw;
}
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count);
foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray())
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
topicCount = topicList.Count;
}
else if (topics.Contains(","))
{
string[] topicArray = topics.Split(new char[] { ',' });
topicCount = topicArray.Length;
sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length);
foreach (var t in topicArray.OrderBy(r => r).ToArray())
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
}
}
else
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics,
zookeeper,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
topicCount = 1;
}
if (topicCount > 1)
{
sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n");
sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_LeaderCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n");
sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_ReplicaCountDistrib)
{
sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
}
string s = sbAll.ToString();
if (dumpToLog)
{
Logger.Info(s);
}
if (dumpToConsole)
{
Console.WriteLine(s);
}
if (!string.IsNullOrEmpty(file))
{
Console.WriteLine("Will write to {0}", file);
using (StreamWriter sw = new StreamWriter(file, false))
{
sw.WriteLine(s);
}
}
}
}
//{"controller_epoch":4,"leader":2,"version":1,"leader_epoch":5,"isr":[2]}
internal static ArrayList GetIsr(ZooKeeperClient zkClient, string topic, int partition)
{
string data = zkClient.ReadData<string>(string.Format("/brokers/topics/{0}/partitions/{1}/state"
, topic, partition), true);
Dictionary<string, object> ctx = new JavaScriptSerializer().Deserialize<Dictionary<string, object>>(data);
Type ty = ctx["isr"].GetType();
return (ArrayList)ctx["isr"];
}
internal static string DumpTopicMetadataAndOffsetInternal(ZooKeeperClient zkClient, string topic,
string zookeeper,
int partitionIndex,
bool includePartitionDetailInfo,
bool includeOffsetInfo,
DateTime timestamp,
SortedDictionary<int, int> parttionBrokerID_LeaderCountDistribAll,
SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistribAll,
SortedDictionary<int, long> latestOffset,
SortedDictionary<int, long> latestLength)
{
StringBuilder sb = new StringBuilder();
string s = string.Empty;
//BrokerID -->Count of as leader
SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
//BrokerID -->Count of as replica
SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
try
{
if (string.IsNullOrEmpty(zookeeper))
{
Logger.Error(" zookeeper should be provided");
sb.AppendFormat(DumpTopicError, topic);
}
else
{
KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration()
{
Zookeeper = zookeeper
};
config.Verify();
Dictionary<int, int[]> detailDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, topic);
using (KafkaSimpleManager<int, Message> kafkaSimpleManager = new KafkaSimpleManager<int, Message>(config))
{
TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(KafkaNETExampleConstants.DefaultVersionId, ClientID, TopicMetadataRequestID++, topic, true);
int partitionCount = topicMetadata.PartitionsMetadata.Count();
sb.AppendFormat("Topic:{0}\tPartitionCount:{1}\t", topic, partitionCount);
int replicationFactor = Enumerable.Count<Broker>(topicMetadata.PartitionsMetadata.First().Replicas);
sb.AppendFormat("ReplicationFactor:{0}\t", replicationFactor);
//TODO: compare detailDataInZookeeper and check which one missed.
StringBuilder sbDetail = new StringBuilder();
if (includePartitionDetailInfo)
{
long sumEndOffset = 0;
long sumLength = 0;
foreach (PartitionMetadata p in topicMetadata.PartitionsMetadata.OrderBy(r => r.PartitionId).ToList())
{
int[] replicaInZookeeper = null;
if (detailDataInZookeeper.ContainsKey(p.PartitionId))
{
replicaInZookeeper = detailDataInZookeeper[p.PartitionId];
detailDataInZookeeper.Remove(p.PartitionId);
}
#region One partition
long earliest = 0;
long latest = 0;
if (partitionIndex == -1 || p.PartitionId == partitionIndex)
{
//sbDetail.AppendFormat("\tTopic:{0}", topic);
sbDetail.AppendFormat("\tPartition:{0}", p.PartitionId);
if (p.Leader != null)
{
sbDetail.AppendFormat("\tLeader:{0}", KafkaConsoleUtil.GetBrokerIDAndIP(p.Leader.Id));
if (parttionBrokerID_LeaderCountDistrib.ContainsKey(p.Leader.Id))
parttionBrokerID_LeaderCountDistrib[p.Leader.Id]++;
else
parttionBrokerID_LeaderCountDistrib.Add(p.Leader.Id, 1);
if (parttionBrokerID_LeaderCountDistribAll.ContainsKey(p.Leader.Id))
parttionBrokerID_LeaderCountDistribAll[p.Leader.Id]++;
else
parttionBrokerID_LeaderCountDistribAll.Add(p.Leader.Id, 1);
}
else
sbDetail.AppendFormat("\tLeader:NoLeader!");
sbDetail.AppendFormat("\tReplicas:{0}", string.Join(",", p.Replicas.Select(r => KafkaConsoleUtil.GetBrokerIDAndIP(r.Id)).ToArray()));
foreach (Broker b in p.Replicas)
{
if (parttionBrokerID_ReplicaCountDistrib.ContainsKey(b.Id))
parttionBrokerID_ReplicaCountDistrib[b.Id]++;
else
parttionBrokerID_ReplicaCountDistrib.Add(b.Id, 1);
if (parttionBrokerID_ReplicaCountDistribAll.ContainsKey(b.Id))
parttionBrokerID_ReplicaCountDistribAll[b.Id]++;
else
parttionBrokerID_ReplicaCountDistribAll.Add(b.Id, 1);
}
//sbDetail.AppendFormat("\tIsr:{0}", string.Join(",", p.Isr.Select(r => r.Id).ToArray()));
ArrayList isrs = GetIsr(zkClient, topic, p.PartitionId);
sbDetail.AppendFormat("\tIsr:{0}", string.Join(",", isrs.ToArray().Select(r => KafkaConsoleUtil.GetBrokerIDAndIP(Convert.ToInt32(r)))));
//TODO: add missed replica
#region Offset
if (includeOffsetInfo)
{
try
{
kafkaSimpleManager.RefreshAndGetOffset(KafkaNETExampleConstants.DefaultVersionId, ClientID, TopicMetadataRequestID++, topic, p.PartitionId, true, out earliest, out latest);
sumEndOffset += latest;
sumLength += (latest - earliest);
sbDetail.AppendFormat("\tlength:{2}\tearliest:{0}\tlatest:{1}"
, earliest
, latest
, (latest - earliest) == 0 ? "(empty)" : (latest - earliest).ToString());
sbDetail.AppendFormat("\r\n");
latestOffset.Add(p.PartitionId, latest);
latestLength.Add(p.PartitionId, latest - earliest);
}
catch (NoLeaderForPartitionException e)
{
sbDetail.AppendFormat(" ERROR:{0}\r\n", e.Message);
}
catch (UnableToConnectToHostException e)
{
sbDetail.AppendFormat(" ERROR:{0}\r\n", e.Message);
}
if (timestamp != DateTime.MinValue)
{
long timestampLong = KafkaClientHelperUtils.ToUnixTimestampMillis(timestamp);
try
{
long timeStampOffset = kafkaSimpleManager.RefreshAndGetOffsetByTimeStamp(KafkaNETExampleConstants.DefaultVersionId, ClientID, TopicMetadataRequestID++, topic, p.PartitionId, timestamp);
sbDetail.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tUnixTimestamp:{2}\t"
, timeStampOffset
, KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s")
, timestampLong);
sbDetail.AppendFormat("\r\n");
}
catch (TimeStampTooSmallException)
{
sbDetail.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tUnixTimestamp:{2}\t"
, "NA since no data before the time you specified, please retry with a bigger value."
, KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s")
, timestampLong);
sbDetail.AppendFormat("\r\n");
}
}
}
#endregion
}
#endregion
}
if (includeOffsetInfo)
{
sb.AppendFormat("SumeEndOffset:{0:0,0} SumLength:{1:0,0}\r\n", sumEndOffset, sumLength);
}
else
{
sb.AppendFormat("\r\n");
}
if (detailDataInZookeeper.Any())
{
foreach (KeyValuePair<int, int[]> kv in detailDataInZookeeper)
{
sb.AppendFormat("=ERROR=MISSED partition= {0} Replicas {1} ", kv.Key, string.Join(",", kv.Value.Select(r => r.ToString()).ToArray()));
}
}
}
sb.Append(sbDetail.ToString());
sb.AppendFormat("\tBroker as leader distribution======={0}=======\r\n", topic);
sb.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_LeaderCountDistrib)
{
sb.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
sb.AppendFormat("\tBroker as replica distribution========={0}=====\r\n", topic);
sb.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
foreach (KeyValuePair<int, int> kv in parttionBrokerID_ReplicaCountDistrib)
{
sb.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
}
sb.AppendFormat("\r\n");
}
}
s = sb.ToString();
}
catch (NoBrokerForTopicException e)
{
sb.AppendFormat("\r\nTopic:{0}\t ==NoBrokerForTopicException:{1}!!!== \r\n", topic, e.Message);
s = sb.ToString();
}
catch (UnableToConnectToHostException e)
{
sb.AppendFormat("\r\nTopic:{0}\t ==UnableToConnectToHostException:{1}!!!== \r\n", topic, e.Message);
s = sb.ToString();
}
catch (Exception ex)
{
Logger.ErrorFormat("Dump topic got exception:{0}\r\ninput parameter:Topic:{1}\tZookeeper:{2}\tKafka:{3}\tPartionIndex:{4}\tincludePartitionDetailInfo:{5}\tincludeOffsetInfo:{6}\ttimestamp:{7}\r\nPartial result:{8}"
, ExceptionUtil.GetExceptionDetailInfo(ex),
topic,
zookeeper,
string.Empty,
partitionIndex,
includePartitionDetailInfo,
includeOffsetInfo,
timestamp,
s);
}
return s;
}
//[zk: localhost(CONNECTED) 12] get /brokers/topics/mvlogs
//{"version":1,"partitions":{"1":[3,2],"0":[2,3]}}
public static Dictionary<int, int[]> GetTopicMetadataInzookeeper(ZooKeeperClient zkClient, string topic)
{
Dictionary<int, int[]> treturn = new Dictionary<int, int[]>();
try
{
string data = zkClient.ReadData<string>(string.Format("/brokers/topics/{0}", topic), true);
Dictionary<string, object> ctx = new JavaScriptSerializer().Deserialize<Dictionary<string, object>>(data);
Type ty = ctx["partitions"].GetType();
//Logger.InfoFormat("The type for partitions :{0}", ty.FullName);
Dictionary<string, object> tpartitons = (Dictionary<string, object>)ctx["partitions"];
foreach (KeyValuePair<string, object> kv in tpartitons)
{
int partitionID = Convert.ToInt32(kv.Key);
//Logger.InfoFormat("The type for partitions value :{0}", kv.Value.GetType().FullName);
ArrayList rep = (ArrayList)kv.Value;
int[] partitionReplicas = new int[rep.Count];
for (int i = 0; i < rep.Count; i++)
partitionReplicas[i] = Convert.ToInt32(rep[i]);
treturn.Add(partitionID, partitionReplicas);
}
Logger.InfoFormat("Get topic data directly from zookeeper Topic:{0} Data:{1} Partition count:{2}", topic, data, treturn.Count);
}
catch (Exception ex)
{
Logger.Error("Failed to get topic " + topic + " data directly from zookeeper: " + ex.FormatException());
}
return treturn;
}
public void WhenStateChangedToConnectedStateListenerFires()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Subscribe(this);
client.Connect();
WaitUntillIdle(client, 500);
}
Assert.AreEqual(1, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
}
public void WhenStateChangedToExpiredStateAndSessionListenersFire()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Subscribe(this);
client.Connect();
WaitUntillIdle(client, 500);
client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
WaitUntillIdle(client, 3000);
}
Assert.AreEqual(4, this.events.Count);
ZooKeeperEventArgs e = this.events[1];
Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.Expired);
e = this.events[2];
Assert.AreEqual(ZooKeeperEventTypes.SessionCreated, e.Type);
Assert.IsInstanceOf<ZooKeeperSessionCreatedEventArgs>(e);
e = this.events[3];
Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
}
public void WhenNewBrokerInTopicIsAddedZKBrokerPartitionInfoUpdatesMappings()
{
var producerConfig = new ProducerConfig(clientConfig);
IDictionary<string, SortedSet<Partition>> mappings;
IDictionary<int, Broker> brokers;
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic;
string topicBrokerPath = topicPath + "/" + 2345;
using (IZooKeeperClient client = new ZooKeeperClient(
producerConfig.ZkConnect,
producerConfig.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
{
brokers = brokerPartitionInfo.GetAllBrokerInfo();
mappings =
ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
"topicBrokerPartitions", brokerPartitionInfo);
client.CreatePersistent(brokerPath, true);
client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
client.CreatePersistent(topicPath, true);
WaitUntillIdle(client, 500);
Assert.IsTrue(brokers.ContainsKey(2345));
Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic));
client.CreatePersistent(topicBrokerPath, true);
client.WriteData(topicBrokerPath, 5);
WaitUntillIdle(client, 500);
client.UnsubscribeAll();
client.DeleteRecursive(brokerPath);
client.DeleteRecursive(topicPath);
}
}
Assert.NotNull(brokers);
Assert.Greater(brokers.Count, 0);
Assert.NotNull(mappings);
Assert.Greater(mappings.Count, 0);
Assert.IsTrue(brokers.ContainsKey(2345));
Assert.IsTrue(mappings.Keys.Contains(CurrentTestTopic));
Assert.AreEqual(5, mappings[CurrentTestTopic].Count);
}
public void ZooKeeperClientChecksIfPathExists()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
Assert.IsTrue(client.Exists(ZooKeeperClient.DefaultBrokerTopicsPath, false));
}
}
public void WhenNewTopicIsAddedZKBrokerPartitionInfoUpdatesMappings()
{
var producerConfig = new ProducerConfig(clientConfig);
IDictionary<string, SortedSet<Partition>> mappings;
string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic;
using (IZooKeeperClient client = new ZooKeeperClient(
producerConfig.ZkConnect,
producerConfig.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
{
mappings =
ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
"topicBrokerPartitions", brokerPartitionInfo);
client.CreatePersistent(topicPath, true);
WaitUntillIdle(client, 500);
client.UnsubscribeAll();
client.DeleteRecursive(topicPath);
}
}
Assert.NotNull(mappings);
Assert.Greater(mappings.Count, 0);
Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic));
}
public void ZooKeeperClientCreatesANewPathAndDeletesIt()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
string myPath = "/" + Guid.NewGuid();
client.CreatePersistent(myPath, false);
Assert.IsTrue(client.Exists(myPath));
client.Delete(myPath);
Assert.IsFalse(client.Exists(myPath));
}
}
public void WhenSessionIsExpiredListenerRecreatesEphemeralNodes()
{
{
var producerConfig = new ProducerConfig(clientConfig);
IDictionary<string, SortedSet<Partition>> mappings;
IDictionary<int, Broker> brokers;
IDictionary<string, SortedSet<Partition>> mappings2;
IDictionary<int, Broker> brokers2;
using (IZooKeeperClient client = new ZooKeeperClient(
producerConfig.ZkConnect,
producerConfig.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
{
brokers = brokerPartitionInfo.GetAllBrokerInfo();
mappings =
ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
"topicBrokerPartitions", brokerPartitionInfo);
Assert.NotNull(brokers);
Assert.Greater(brokers.Count, 0);
Assert.NotNull(mappings);
Assert.Greater(mappings.Count, 0);
client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
WaitUntillIdle(client, 3000);
brokers2 = brokerPartitionInfo.GetAllBrokerInfo();
mappings2 =
ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
"topicBrokerPartitions", brokerPartitionInfo);
}
}
Assert.NotNull(brokers2);
Assert.Greater(brokers2.Count, 0);
Assert.NotNull(mappings2);
Assert.Greater(mappings2.Count, 0);
Assert.AreEqual(brokers.Count, brokers2.Count);
Assert.AreEqual(mappings.Count, mappings2.Count);
}
}
public void ZooKeeperClientFailsWhenCreatedWithWrongConnectionInfo()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client = new ZooKeeperClient(
"random text",
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
Assert.Throws<FormatException>(client.Connect);
}
}
public void WhenBrokerIsRemovedZKBrokerPartitionInfoUpdatesBrokersList()
{
var producerConfig = new ProducerConfig(clientConfig);
IDictionary<int, Broker> brokers;
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
using (IZooKeeperClient client = new ZooKeeperClient(
producerConfig.ZkConnect,
producerConfig.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
{
brokers = brokerPartitionInfo.GetAllBrokerInfo();
client.CreatePersistent(brokerPath, true);
client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
WaitUntillIdle(client, 500);
Assert.NotNull(brokers);
Assert.Greater(brokers.Count, 0);
Assert.IsTrue(brokers.ContainsKey(2345));
client.DeleteRecursive(brokerPath);
WaitUntillIdle(client, 500);
}
}
Assert.NotNull(brokers);
Assert.Greater(brokers.Count, 0);
Assert.IsFalse(brokers.ContainsKey(2345));
}
private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List<ConsumeGroupMonitorUnit> units)
{
Logger.Info("Will refresh all consume group from zk ...");
string path = "/consumers";
//TODO: add /users
IEnumerable<string> groups = zkClient.GetChildren(path);
foreach (var g in groups)
{
Logger.InfoFormat("Will check group {0}", g);
string topicpath = string.Format("/consumers/{0}/owners", g);
IEnumerable<string> topics = zkClient.GetChildren(topicpath);
foreach (var t in topics)
{
Logger.InfoFormat("Will check topic {0}", t);
units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t));
}
}
Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count);
}
public void WhenClientWillNotConnectWithinGivenTimeThrows()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
using (IZooKeeperClient client =
new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer,
1))
{
client.Connect();
}
}
internal bool Run(ZooKeeperClient zkClient, string zookeeper)
{
bool success = true;
SortedDictionary<int, long> latestCommited = GetComsumerGroupOffsets(zkClient, this.topic, this.group);
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
//BrokerID -->Count of as replica
SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
//Owner
SortedDictionary<int, string> latestOwners = GetComsumerGroupOwners(zkClient, this.topic, this.group);
TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, this.topic,
zookeeper,
-1,
true,
true,
DateTime.MinValue,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength);
if (latestOffsetDictLastValue == null)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("====Partitions====\r\n");
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Key);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====LatestOffset====\r\n");
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====ConsumedOffset====\r\n");
foreach (KeyValuePair<int, long> kv in latestCommited)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("===Latest-Earliest: Initial value====\r\n");
foreach (KeyValuePair<int, long> kv in latestLength)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====Latest-Commited: Initial Value====\r\n");
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
if (latestCommited.ContainsKey(kv.Key))
sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]);
else
sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
if (latestOffsetDict.Count == latestOwners.Count)
{
sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count);
}
else
{
sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count);
}
foreach (var ownerByOwnerName in (from o in latestOwners
group o by o.Value into g
select new
{
owner = g.Key,
partitions = g.ToArray()
}).OrderBy(r => r.owner))
{
sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length);
for (int k = 0; k < ownerByOwnerName.partitions.Length; k++)
{
if (k == 0)
sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]);
else
sb.AppendFormat(", {0}", ownerByOwnerName.partitions[k]);
}
sb.AppendFormat("\r\n");
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString());
}
else
{
//Length
StringBuilder sb = new StringBuilder();
sb.Append("Latest-Earliest: ");
foreach (KeyValuePair<int, long> kv in latestLength)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Latest Delta
sb = new StringBuilder();
long latestDelta = 0;
long aggregateLatestDelta = 0;
foreach (KeyValuePair<int, long> kv in latestOffsetDictLastValue)
{
if (latestOffsetDict.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value);
latestDelta += latestOffsetDict[kv.Key] - kv.Value;
}
else
{
sb.AppendFormat("Latest:{0,-9} ", kv.Value);
}
if (latestOffsetDictFirstValue.ContainsKey(kv.Key))
{
aggregateLatestDelta += kv.Value - latestOffsetDictFirstValue[kv.Key];
}
}
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
if (!latestOffsetDictLastValue.ContainsKey(kv.Key))
sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value);
}
sb.Insert(0, string.Format("Latest Delta: {0}", latestDelta));
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Commited Delta
sb = new StringBuilder();
long latestDeltaCommited = 0;
long aggregateLatestCommite = 0;
foreach (KeyValuePair<int, long> kv in latestCommitedDictLastValue)
{
if (latestCommited.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", latestCommited[kv.Key] - kv.Value);
latestDeltaCommited += latestCommited[kv.Key] - kv.Value;
}
else
{
sb.AppendFormat("Commited:{0,-9} ", kv.Value);
}
if (latestCommitedDictFirstValue.ContainsKey(kv.Key))
{
aggregateLatestCommite += kv.Value - latestCommitedDictFirstValue[kv.Key];
}
}
foreach (KeyValuePair<int, long> kv in latestCommited)
{
if (!latestCommitedDictLastValue.ContainsKey(kv.Key))
sb.AppendFormat("NewCommited:{0}-{1,-9} ", kv.Key, kv.Value);
}
sb.Insert(0, string.Format("Commited Delta: {0}", latestDeltaCommited));
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Gap
sb = new StringBuilder();
sb.AppendFormat("Latest-Commited: {0}= ", latestOffsetDict.Count);
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
if (latestCommited.ContainsKey(kv.Key))
sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]);
else
sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Owner
sb = new StringBuilder();
if (latestOffsetDict.Count == latestOwners.Count)
{
sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count);
}
else
{
sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count);
success = false;
}
foreach (var ownerByOwnerName in (from o in latestOwners
group o by o.Value into g
select new
{
owner = g.Key,
partitions = g.ToArray()
}).OrderBy(r => r.owner))
{
sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length);
for (int k = 0; k < ownerByOwnerName.partitions.Length; k++)
{
if (k == 0)
sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]);
else
sb.AppendFormat(", {0}", ownerByOwnerName.partitions[k]);
}
sb.AppendFormat("\r\n");
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("In last {0:0.0} seconds. Totally latest offset change:{1} Totally commited offset change:{2} . Percentage:{3:P2} Time:{4}\r\n"
, (DateTime.UtcNow - startTime).TotalSeconds, aggregateLatestDelta, aggregateLatestCommite, aggregateLatestCommite * 1.0 / aggregateLatestDelta, DateTime.Now);
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
}
previousOwners = latestOwners;
latestOffsetDictLastValue = latestOffsetDict;
latestCommitedDictLastValue = latestCommited;
if (latestOffsetDictFirstValue == null)
latestOffsetDictFirstValue = latestOffsetDict;
if (latestCommitedDictFirstValue == null)
latestCommitedDictFirstValue = latestCommited;
return success;
}
public void WhenDataChangedDataListenerFires()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
string myPath = "/" + Guid.NewGuid();
string sourceData = "my test data";
string resultData;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
client.CreatePersistent(myPath, true);
WaitUntillIdle(client, 500);
client.Subscribe(myPath, this as IZooKeeperDataListener);
client.Subscribe(myPath, this as IZooKeeperChildListener);
client.WriteData(myPath, sourceData);
WaitUntillIdle(client, 500);
client.UnsubscribeAll();
resultData = client.ReadData<string>(myPath);
client.Delete(myPath);
}
Assert.IsTrue(!string.IsNullOrEmpty(resultData));
Assert.AreEqual(sourceData, resultData);
Assert.AreEqual(1, this.events.Count);
ZooKeeperEventArgs e = this.events[0];
Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
Assert.IsInstanceOf<ZooKeeperDataChangedEventArgs>(e);
Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
Assert.IsNotNull(((ZooKeeperDataChangedEventArgs)e).Data);
Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Data, sourceData);
}
internal static void Run(ProduceMonitorHelperOptions produceMonitorOptions)
{
using (ZooKeeperClient zkClient = new ZooKeeperClient(produceMonitorOptions.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
while (true)
{
SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
//BrokerID -->Count of as replica
SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, produceMonitorOptions.Topic,
produceMonitorOptions.Zookeeper,
-1,
true,
true,
DateTime.MinValue,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength);
if (latestOffsetDictLastValue == null)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("====Partitions====\r\n");
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Key);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====LatestOffset====\r\n");
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
}
else
{
StringBuilder sb = new StringBuilder();
sb.Append("Latest Delta: ");
foreach (KeyValuePair<int, long> kv in latestOffsetDictLastValue)
{
if (latestOffsetDict.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value);
}
else
{
sb.AppendFormat("Latest:{0,-9} ", kv.Value);
}
}
foreach (KeyValuePair<int, long> kv in latestOffsetDict)
{
if (!latestOffsetDictLastValue.ContainsKey(kv.Key))
sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
}
latestOffsetDictLastValue = latestOffsetDict;
Thread.Sleep(produceMonitorOptions.IntervalInSeconds * 1000);
}
}
}
public void WhenSessionExpiredClientReconnects()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
IZooKeeperConnection conn1;
IZooKeeperConnection conn2;
using (IZooKeeperClient client = new ZooKeeperClient(
prodConfig.ZooKeeper.ZkConnect,
prodConfig.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer))
{
client.Connect();
conn1 = ReflectionHelper.GetInstanceField<ZooKeeperConnection>("connection", client);
client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
WaitUntillIdle(client, 1000);
conn2 = ReflectionHelper.GetInstanceField<ZooKeeperConnection>("connection", client);
}
Assert.AreNotEqual(conn1, conn2);
}