internal bool Run(ZooKeeperClient zkClient, string zookeeper) { bool success = true; SortedDictionary <int, long> latestCommited = GetComsumerGroupOffsets(zkClient, this.topic, this.group); SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>(); //BrokerID -->Count of as replica SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); //Owner SortedDictionary <int, string> latestOwners = GetComsumerGroupOwners(zkClient, this.topic, this.group); TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, this.topic, zookeeper, -1, true, true, DateTime.MinValue, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength); if (latestOffsetDictLastValue == null) { StringBuilder sb = new StringBuilder(); sb.AppendFormat("====Partitions====\r\n"); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { sb.AppendFormat("{0,-9} ", kv.Key); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); sb = new StringBuilder(); sb.AppendFormat("====LatestOffset====\r\n"); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { sb.AppendFormat("{0,-9} ", kv.Value); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); sb = new StringBuilder(); sb.AppendFormat("====ConsumedOffset====\r\n"); foreach (KeyValuePair <int, long> kv in latestCommited) { sb.AppendFormat("{0,-9} ", kv.Value); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); sb = new StringBuilder(); sb.AppendFormat("===Latest-Earliest: Initial value====\r\n"); foreach (KeyValuePair <int, long> kv in latestLength) { sb.AppendFormat("{0,-9} ", kv.Value); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); sb = new StringBuilder(); sb.AppendFormat("====Latest-Commited: Initial Value====\r\n"); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { if (latestCommited.ContainsKey(kv.Key)) { sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]); } else { sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value); } } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); sb = new StringBuilder(); if (latestOffsetDict.Count == latestOwners.Count) { sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count); } else { sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count); } foreach (var ownerByOwnerName in (from o in latestOwners group o by o.Value into g select new { owner = g.Key, partitions = g.ToArray() }).OrderBy(r => r.owner)) { sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length); for (int k = 0; k < ownerByOwnerName.partitions.Length; k++) { if (k == 0) { sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]); } else { sb.AppendFormat(", {0}", ownerByOwnerName.partitions[k]); } } sb.AppendFormat("\r\n"); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString()); } else { //Length StringBuilder sb = new StringBuilder(); sb.Append("Latest-Earliest: "); foreach (KeyValuePair <int, long> kv in latestLength) { sb.AppendFormat("{0,-9} ", kv.Value); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); //Latest Delta sb = new StringBuilder(); long latestDelta = 0; long aggregateLatestDelta = 0; foreach (KeyValuePair <int, long> kv in latestOffsetDictLastValue) { if (latestOffsetDict.ContainsKey(kv.Key)) { sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value); latestDelta += latestOffsetDict[kv.Key] - kv.Value; } else { sb.AppendFormat("Latest:{0,-9} ", kv.Value); } if (latestOffsetDictFirstValue.ContainsKey(kv.Key)) { aggregateLatestDelta += kv.Value - latestOffsetDictFirstValue[kv.Key]; } } foreach (KeyValuePair <int, long> kv in latestOffsetDict) { if (!latestOffsetDictLastValue.ContainsKey(kv.Key)) { sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value); } } sb.Insert(0, string.Format("Latest Delta: {0}", latestDelta)); Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); //Commited Delta sb = new StringBuilder(); long latestDeltaCommited = 0; long aggregateLatestCommite = 0; foreach (KeyValuePair <int, long> kv in latestCommitedDictLastValue) { if (latestCommited.ContainsKey(kv.Key)) { sb.AppendFormat("{0,-9} ", latestCommited[kv.Key] - kv.Value); latestDeltaCommited += latestCommited[kv.Key] - kv.Value; } else { sb.AppendFormat("Commited:{0,-9} ", kv.Value); } if (latestCommitedDictFirstValue.ContainsKey(kv.Key)) { aggregateLatestCommite += kv.Value - latestCommitedDictFirstValue[kv.Key]; } } foreach (KeyValuePair <int, long> kv in latestCommited) { if (!latestCommitedDictLastValue.ContainsKey(kv.Key)) { sb.AppendFormat("NewCommited:{0}-{1,-9} ", kv.Key, kv.Value); } } sb.Insert(0, string.Format("Commited Delta: {0}", latestDeltaCommited)); Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); //Gap sb = new StringBuilder(); sb.AppendFormat("Latest-Commited: {0}= ", latestOffsetDict.Count); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { if (latestCommited.ContainsKey(kv.Key)) { sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]); } else { sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value); } } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); //Owner sb = new StringBuilder(); if (latestOffsetDict.Count == latestOwners.Count) { sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count); } else { sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count); success = false; } foreach (var ownerByOwnerName in (from o in latestOwners group o by o.Value into g select new { owner = g.Key, partitions = g.ToArray() }).OrderBy(r => r.owner)) { sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length); for (int k = 0; k < ownerByOwnerName.partitions.Length; k++) { if (k == 0) { sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]); } else { sb.AppendFormat(", {0}", ownerByOwnerName.partitions[k]); } } sb.AppendFormat("\r\n"); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString()); sb = new StringBuilder(); sb.AppendFormat("In last {0:0.0} seconds. Totally latest offset change:{1} Totally commited offset change:{2} . Percentage:{3:P2} Time:{4}\r\n" , (DateTime.UtcNow - startTime).TotalSeconds, aggregateLatestDelta, aggregateLatestCommite, aggregateLatestCommite * 1.0 / aggregateLatestDelta, DateTime.Now); Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString()); } previousOwners = latestOwners; latestOffsetDictLastValue = latestOffsetDict; latestCommitedDictLastValue = latestCommited; if (latestOffsetDictFirstValue == null) { latestOffsetDictFirstValue = latestOffsetDict; } if (latestCommitedDictFirstValue == null) { latestCommitedDictFirstValue = latestCommited; } return(success); }
internal static void Run(ProduceMonitorHelperOptions produceMonitorOptions) { using (ZooKeeperClient zkClient = new ZooKeeperClient(produceMonitorOptions.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); while (true) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>(); //BrokerID -->Count of as replica SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, produceMonitorOptions.Topic, produceMonitorOptions.Zookeeper, -1, true, true, DateTime.MinValue, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength); if (latestOffsetDictLastValue == null) { StringBuilder sb = new StringBuilder(); sb.AppendFormat("====Partitions====\r\n"); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { sb.AppendFormat("{0,-9} ", kv.Key); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString()); sb = new StringBuilder(); sb.AppendFormat("====LatestOffset====\r\n"); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { sb.AppendFormat("{0,-9} ", kv.Value); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString()); } else { StringBuilder sb = new StringBuilder(); sb.Append("Latest Delta: "); foreach (KeyValuePair <int, long> kv in latestOffsetDictLastValue) { if (latestOffsetDict.ContainsKey(kv.Key)) { sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value); } else { sb.AppendFormat("Latest:{0,-9} ", kv.Value); } } foreach (KeyValuePair <int, long> kv in latestOffsetDict) { if (!latestOffsetDictLastValue.ContainsKey(kv.Key)) { sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value); } } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString()); } latestOffsetDictLastValue = latestOffsetDict; Thread.Sleep(produceMonitorOptions.IntervalInSeconds * 1000); } } }