internal bool Run(ZooKeeperClient zkClient, string zookeeper)
{
bool success = true;
SortedDictionary <int, long> latestCommited = GetComsumerGroupOffsets(zkClient, this.topic, this.group);
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>();
//BrokerID -->Count of as replica
SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
//Owner
SortedDictionary <int, string> latestOwners = GetComsumerGroupOwners(zkClient, this.topic, this.group);
TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, this.topic,
zookeeper,
-1,
true,
true,
DateTime.MinValue,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength);
if (latestOffsetDictLastValue == null)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("====Partitions====\r\n");
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Key);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====LatestOffset====\r\n");
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====ConsumedOffset====\r\n");
foreach (KeyValuePair <int, long> kv in latestCommited)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("===Latest-Earliest: Initial value====\r\n");
foreach (KeyValuePair <int, long> kv in latestLength)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====Latest-Commited: Initial Value====\r\n");
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
if (latestCommited.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]);
}
else
{
sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value);
}
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
if (latestOffsetDict.Count == latestOwners.Count)
{
sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count);
}
else
{
sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count);
}
foreach (var ownerByOwnerName in (from o in latestOwners
group o by o.Value into g
select new
{
owner = g.Key,
partitions = g.ToArray()
}).OrderBy(r => r.owner))
{
sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length);
for (int k = 0; k < ownerByOwnerName.partitions.Length; k++)
{
if (k == 0)
{
sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]);
}
else
{
sb.AppendFormat(", {0}", ownerByOwnerName.partitions[k]);
}
}
sb.AppendFormat("\r\n");
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString());
}
else
{
//Length
StringBuilder sb = new StringBuilder();
sb.Append("Latest-Earliest: ");
foreach (KeyValuePair <int, long> kv in latestLength)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Latest Delta
sb = new StringBuilder();
long latestDelta = 0;
long aggregateLatestDelta = 0;
foreach (KeyValuePair <int, long> kv in latestOffsetDictLastValue)
{
if (latestOffsetDict.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value);
latestDelta += latestOffsetDict[kv.Key] - kv.Value;
}
else
{
sb.AppendFormat("Latest:{0,-9} ", kv.Value);
}
if (latestOffsetDictFirstValue.ContainsKey(kv.Key))
{
aggregateLatestDelta += kv.Value - latestOffsetDictFirstValue[kv.Key];
}
}
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
if (!latestOffsetDictLastValue.ContainsKey(kv.Key))
{
sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value);
}
}
sb.Insert(0, string.Format("Latest Delta: {0}", latestDelta));
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Commited Delta
sb = new StringBuilder();
long latestDeltaCommited = 0;
long aggregateLatestCommite = 0;
foreach (KeyValuePair <int, long> kv in latestCommitedDictLastValue)
{
if (latestCommited.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", latestCommited[kv.Key] - kv.Value);
latestDeltaCommited += latestCommited[kv.Key] - kv.Value;
}
else
{
sb.AppendFormat("Commited:{0,-9} ", kv.Value);
}
if (latestCommitedDictFirstValue.ContainsKey(kv.Key))
{
aggregateLatestCommite += kv.Value - latestCommitedDictFirstValue[kv.Key];
}
}
foreach (KeyValuePair <int, long> kv in latestCommited)
{
if (!latestCommitedDictLastValue.ContainsKey(kv.Key))
{
sb.AppendFormat("NewCommited:{0}-{1,-9} ", kv.Key, kv.Value);
}
}
sb.Insert(0, string.Format("Commited Delta: {0}", latestDeltaCommited));
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Gap
sb = new StringBuilder();
sb.AppendFormat("Latest-Commited: {0}= ", latestOffsetDict.Count);
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
if (latestCommited.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]);
}
else
{
sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value);
}
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
//Owner
sb = new StringBuilder();
if (latestOffsetDict.Count == latestOwners.Count)
{
sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count);
}
else
{
sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count);
success = false;
}
foreach (var ownerByOwnerName in (from o in latestOwners
group o by o.Value into g
select new
{
owner = g.Key,
partitions = g.ToArray()
}).OrderBy(r => r.owner))
{
sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length);
for (int k = 0; k < ownerByOwnerName.partitions.Length; k++)
{
if (k == 0)
{
sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]);
}
else
{
sb.AppendFormat(", {0}", ownerByOwnerName.partitions[k]);
}
}
sb.AppendFormat("\r\n");
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("In last {0:0.0} seconds. Totally latest offset change:{1} Totally commited offset change:{2} . Percentage:{3:P2} Time:{4}\r\n"
, (DateTime.UtcNow - startTime).TotalSeconds, aggregateLatestDelta, aggregateLatestCommite, aggregateLatestCommite * 1.0 / aggregateLatestDelta, DateTime.Now);
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
}
previousOwners = latestOwners;
latestOffsetDictLastValue = latestOffsetDict;
latestCommitedDictLastValue = latestCommited;
if (latestOffsetDictFirstValue == null)
{
latestOffsetDictFirstValue = latestOffsetDict;
}
if (latestCommitedDictFirstValue == null)
{
latestCommitedDictFirstValue = latestCommited;
}
return(success);
}
internal static void Run(ProduceMonitorHelperOptions produceMonitorOptions)
{
using (ZooKeeperClient zkClient = new ZooKeeperClient(produceMonitorOptions.Zookeeper,
ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
{
zkClient.Connect();
while (true)
{
SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>();
SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>();
//BrokerID -->Count of as replica
SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>();
SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>();
TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, produceMonitorOptions.Topic,
produceMonitorOptions.Zookeeper,
-1,
true,
true,
DateTime.MinValue,
parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength);
if (latestOffsetDictLastValue == null)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("====Partitions====\r\n");
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Key);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
sb = new StringBuilder();
sb.AppendFormat("====LatestOffset====\r\n");
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
sb.AppendFormat("{0,-9} ", kv.Value);
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
}
else
{
StringBuilder sb = new StringBuilder();
sb.Append("Latest Delta: ");
foreach (KeyValuePair <int, long> kv in latestOffsetDictLastValue)
{
if (latestOffsetDict.ContainsKey(kv.Key))
{
sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value);
}
else
{
sb.AppendFormat("Latest:{0,-9} ", kv.Value);
}
}
foreach (KeyValuePair <int, long> kv in latestOffsetDict)
{
if (!latestOffsetDictLastValue.ContainsKey(kv.Key))
{
sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value);
}
}
Logger.Info(sb.ToString());
KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
}
latestOffsetDictLastValue = latestOffsetDict;
Thread.Sleep(produceMonitorOptions.IntervalInSeconds * 1000);
}
}
}