public void WhenStateChangedToExpiredStateAndSessionListenersFire() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Subscribe(this); client.Connect(); WaitUntillIdle(client, 500); client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null)); WaitUntillIdle(client, 3000); } Assert.AreEqual(4, this.events.Count); ZooKeeperEventArgs e = this.events[1]; Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type); Assert.IsInstanceOf <ZooKeeperStateChangedEventArgs>(e); Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.Expired); e = this.events[2]; Assert.AreEqual(ZooKeeperEventTypes.SessionCreated, e.Type); Assert.IsInstanceOf <ZooKeeperSessionCreatedEventArgs>(e); e = this.events[3]; Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type); Assert.IsInstanceOf <ZooKeeperStateChangedEventArgs>(e); Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected); }
public void WhenChildIsDeletedChildListenerOnParentFires() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; string myPath = "/" + Guid.NewGuid(); using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); client.CreatePersistent(myPath, true); WaitUntillIdle(client, 500); client.Subscribe("/", this as IZooKeeperChildListener); client.Delete(myPath); WaitUntillIdle(client, 500); } Assert.AreEqual(1, this.events.Count); ZooKeeperEventArgs e = this.events[0]; Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type); Assert.IsInstanceOf <ZooKeeperChildChangedEventArgs>(e); Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, "/"); Assert.Greater(((ZooKeeperChildChangedEventArgs)e).Children.Count, 0); Assert.IsFalse(((ZooKeeperChildChangedEventArgs)e).Children.Contains(myPath.Replace("/", string.Empty))); }
public void WhenZNodeIsDeletedChildAndDataDeletedListenersFire() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; string myPath = "/" + Guid.NewGuid(); using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); client.CreatePersistent(myPath, true); WaitUntillIdle(client, 500); client.Subscribe(myPath, this as IZooKeeperChildListener); client.Subscribe(myPath, this as IZooKeeperDataListener); client.Delete(myPath); WaitUntillIdle(client, 500); } Assert.AreEqual(2, this.events.Count); ZooKeeperEventArgs e = this.events[0]; Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type); Assert.IsInstanceOf <ZooKeeperChildChangedEventArgs>(e); Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, myPath); Assert.IsNull(((ZooKeeperChildChangedEventArgs)e).Children); e = this.events[1]; Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type); Assert.IsInstanceOf <ZooKeeperDataChangedEventArgs>(e); Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath); Assert.IsNull(((ZooKeeperDataChangedEventArgs)e).Data); }
internal static void Run(ProduceSimpleHelperOption produceOptions) { failedMessageCount = 0; successMessageCount = 0; sentBatchCount = 0; produceMessagePerPartition = new Dictionary <int, int>(); produceMessagePerPartitionExpect = new Dictionary <int, int>(); PrepareSentMessages(produceOptions); kafkaSimpleManagerConfig = new KafkaSimpleManagerConfiguration() { Zookeeper = produceOptions.Zookeeper, MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize, PartitionerClass = produceOptions.PartitionerClass }; kafkaSimpleManagerConfig.Verify(); producerConfigTemplate = new ProducerConfiguration( new List <BrokerConfiguration>() { }) //The Brokers will be replaced inside of KafkaSimpleManager { ForceToPartition = -1, PartitionerClass = kafkaSimpleManagerConfig.PartitionerClass, TotalNumPartitions = 0, RequiredAcks = produceOptions.RequiredAcks, AckTimeout = produceOptions.AckTimeout, SendTimeout = produceOptions.SendTimeout, ReceiveTimeout = produceOptions.ReceiveTimeout, CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()), BufferSize = produceOptions.BufferSize, SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize) }; using (ZooKeeperClient zkClient = new ZooKeeperClient(produceOptions.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); Dictionary <int, int[]> topicDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, produceOptions.Topic); // -2 by default or customized partitioner class. (you need specify PartitionerClass by -l) if (produceOptions.PartitionId == -2) { if (string.IsNullOrEmpty(kafkaSimpleManagerConfig.PartitionerClass)) { throw new ArgumentException("The partitioer class must not be empty if you want to send to partition by partitioner."); } //if (producerConfigTemplate.TotalNumPartitions <= 0) // throw new ArgumentException(string.Format("Please also specify the TotalNumPartitions if you want to send to partition by partitioner.")); ProduceByPartitionerClass(produceOptions, topicDataInZookeeper.Count); } else { ProduceToRandomOrSpecificPartition(produceOptions); } } }
public ZookeeperClient(string zkConnect) { _client = new ZooKeeperClient( zkConnect, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer ); _client.Connect(); }
public void WhenNewBrokerInTopicIsAddedBrokerTopicsListenerUpdatesMappings() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; IDictionary <string, SortedSet <Partition> > mappings; IDictionary <int, Broker> brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic; string topicBrokerPath = topicPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) { brokers = brokerPartitionInfo.GetAllBrokerInfo(); mappings = ReflectionHelper.GetInstanceField <IDictionary <string, SortedSet <Partition> > >( "topicBrokerPartitions", brokerPartitionInfo); } } Assert.NotNull(brokers); Assert.Greater(brokers.Count, 0); Assert.NotNull(mappings); Assert.Greater(mappings.Count, 0); using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); WaitUntillIdle(client, 500); var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null); client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener); client.Subscribe(ZooKeeperClient.DefaultBrokerTopicsPath, brokerTopicsListener); client.CreatePersistent(brokerPath, true); client.WriteData(brokerPath, "172.16.66.2-1310449279123:172.16.66.2:9092"); client.CreatePersistent(topicPath, true); WaitUntillIdle(client, 500); Assert.IsTrue(brokers.ContainsKey(2345)); Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic)); client.CreatePersistent(topicBrokerPath, true); client.WriteData(topicBrokerPath, 5); WaitUntillIdle(client, 500); client.UnsubscribeAll(); WaitUntillIdle(client, 500); client.DeleteRecursive(brokerPath); client.DeleteRecursive(topicPath); } Assert.IsTrue(brokers.ContainsKey(2345)); Assert.IsTrue(mappings.Keys.Contains(CurrentTestTopic)); Assert.AreEqual(5, mappings[CurrentTestTopic].Count); }
public void ZooKeeperClientChecksIfPathExists() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); Assert.IsTrue(client.Exists(ZooKeeperClient.DefaultBrokerTopicsPath, false)); } }
public void ConsumerPorformsRebalancingOnStart() { var config = this.ZooKeeperBasedConsumerConfig; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField <ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary <string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); IList <string> children = client.GetChildren("/consumers", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("group1")); children = client.GetChildren("/consumers/group1", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("ids")); Assert.That(children, Contains.Item("owners")); children = client.GetChildren("/consumers/group1/ids", false); Assert.That(children, Is.Not.Null.And.Not.Empty); string consumerId = children[0]; children = client.GetChildren("/consumers/group1/owners", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(1)); Assert.That(children, Contains.Item("test")); children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(2)); string partId = children[0]; var data = client.ReadData <string>("/consumers/group1/owners/test/" + partId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Contains.Substring(consumerId)); data = client.ReadData <string>("/consumers/group1/ids/" + consumerId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Is.EqualTo("{ \"test\": 1 }")); } using (var client = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); //// Should be created as ephemeral IList <string> children = client.GetChildren("/consumers/group1/ids"); Assert.That(children, Is.Null.Or.Empty); //// Should be created as ephemeral children = client.GetChildren("/consumers/group1/owners/test"); Assert.That(children, Is.Null.Or.Empty); } }
public SyncProducerPool(ProducerConfiguration config) { syncProducers = new ConcurrentDictionary <int, SyncProducerWrapper>(); Config = config; if (config.ZooKeeper != null) { zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer); zkClient.Connect(); } AddProducers(config); }
public void WhenClientWillNotConnectWithinGivenTimeThrows() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer, 1)) { client.Connect(); } }
public void ZooKeeperClientCreateWorkerThreadsOnBeingCreated() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); var eventWorker = ReflectionHelper.GetInstanceField <Thread>("eventWorker", client); var zooKeeperWorker = ReflectionHelper.GetInstanceField <Thread>("zooKeeperEventWorker", client); Assert.NotNull(eventWorker); Assert.NotNull(zooKeeperWorker); } }
public void ZooKeeperClientCreatesANewPathAndDeletesIt() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); string myPath = "/" + Guid.NewGuid(); client.CreatePersistent(myPath, false); Assert.IsTrue(client.Exists(myPath)); client.Delete(myPath); Assert.IsFalse(client.Exists(myPath)); } }
public SyncProducerPool(ProducerConfiguration config, List <ISyncProducer> producers) { syncProducers = new ConcurrentDictionary <int, SyncProducerWrapper>(); Config = config; if (config.ZooKeeper != null) { zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer); zkClient.Connect(); } if (producers != null && producers.Any()) { producers.ForEach(x => syncProducers.TryAdd(x.Config.BrokerId, new SyncProducerWrapper(x, config.SyncProducerOfOneBroker))); } }
public void WhenBrokerIsRemovedBrokerTopicsListenerUpdatesBrokersList() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; IDictionary <string, SortedSet <Partition> > mappings; IDictionary <int, Broker> brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) { brokers = brokerPartitionInfo.GetAllBrokerInfo(); mappings = ReflectionHelper.GetInstanceField <IDictionary <string, SortedSet <Partition> > >( "topicBrokerPartitions", brokerPartitionInfo); } } Assert.NotNull(brokers); Assert.Greater(brokers.Count, 0); Assert.NotNull(mappings); Assert.Greater(mappings.Count, 0); using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); WaitUntillIdle(client, 500); var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null); client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener); client.CreatePersistent(brokerPath, true); client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); WaitUntillIdle(client, 500); Assert.IsTrue(brokers.ContainsKey(2345)); client.DeleteRecursive(brokerPath); WaitUntillIdle(client, 500); Assert.IsFalse(brokers.ContainsKey(2345)); } }
public void WhenStateChangedToConnectedStateListenerFires() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Subscribe(this); client.Connect(); WaitUntillIdle(client, 500); } Assert.AreEqual(1, this.events.Count); ZooKeeperEventArgs e = this.events[0]; Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type); Assert.IsInstanceOf <ZooKeeperStateChangedEventArgs>(e); Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected); }
public void ZooKeeperClientCreatesAChildAndGetsChildren() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); string child = Guid.NewGuid().ToString(); string myPath = "/" + child; client.CreatePersistent(myPath, false); IList <string> children = client.GetChildren("/", false); int countChildren = client.CountChildren("/"); Assert.Greater(children.Count, 0); Assert.AreEqual(children.Count, countChildren); Assert.IsTrue(children.Contains(child)); client.Delete(myPath); } }
public void WhenSessionExpiredClientReconnects() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; IZooKeeperConnection conn1; IZooKeeperConnection conn2; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); conn1 = ReflectionHelper.GetInstanceField <ZooKeeperConnection>("connection", client); client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null)); WaitUntillIdle(client, 1000); conn2 = ReflectionHelper.GetInstanceField <ZooKeeperConnection>("connection", client); } Assert.AreNotEqual(conn1, conn2); }
public void WhenDataChangedDataListenerFires() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; string myPath = "/" + Guid.NewGuid(); string sourceData = "my test data"; string resultData; using (IZooKeeperClient client = new ZooKeeperClient( prodConfig.ZooKeeper.ZkConnect, prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); client.CreatePersistent(myPath, true); WaitUntillIdle(client, 500); client.Subscribe(myPath, this as IZooKeeperDataListener); client.Subscribe(myPath, this as IZooKeeperChildListener); client.WriteData(myPath, sourceData); WaitUntillIdle(client, 500); client.UnsubscribeAll(); resultData = client.ReadData <string>(myPath); client.Delete(myPath); } Assert.IsTrue(!string.IsNullOrEmpty(resultData)); Assert.AreEqual(sourceData, resultData); Assert.AreEqual(1, this.events.Count); ZooKeeperEventArgs e = this.events[0]; Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type); Assert.IsInstanceOf <ZooKeeperDataChangedEventArgs>(e); Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath); Assert.IsNotNull(((ZooKeeperDataChangedEventArgs)e).Data); Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Data, sourceData); }
internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption) { List <ConsumeGroupMonitorUnit> units = new List <ConsumeGroupMonitorUnit>(); if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray)) { if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName)) { units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic)); } else { danymicAllConsumeGroupTopic = true; } } else { string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); foreach (var cg in cgs) { string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); if (temp.Length != 2) { Logger.ErrorFormat("Wrong parameter, Exmaple G1:t1,G2:t2. The value:{0} is invalid.", cg); } else { units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1])); } } } long count = 0; int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds; using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); while (true) { DateTime time = DateTime.Now; string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day); if (!Directory.Exists(dateFolder)) { Directory.CreateDirectory(dateFolder); } if (danymicAllConsumeGroupTopic && count % cycle == 0) { units = new List <ConsumeGroupMonitorUnit>(); RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units); } int countSuccess = 0; foreach (var unit in units) { unit.subFolder = dateFolder; if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper)) { countSuccess++; } } if (countSuccess == units.Count) { Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess); } else { Logger.ErrorFormat("===========All {0} consume group only {1} succ. FAIL=================", units.Count, countSuccess); } Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000); count++; } } }
internal static void Run(ProduceMonitorHelperOptions produceMonitorOptions) { using (ZooKeeperClient zkClient = new ZooKeeperClient(produceMonitorOptions.Zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); while (true) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>(); //BrokerID -->Count of as replica SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, produceMonitorOptions.Topic, produceMonitorOptions.Zookeeper, -1, true, true, DateTime.MinValue, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength); if (latestOffsetDictLastValue == null) { StringBuilder sb = new StringBuilder(); sb.AppendFormat("====Partitions====\r\n"); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { sb.AppendFormat("{0,-9} ", kv.Key); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString()); sb = new StringBuilder(); sb.AppendFormat("====LatestOffset====\r\n"); foreach (KeyValuePair <int, long> kv in latestOffsetDict) { sb.AppendFormat("{0,-9} ", kv.Value); } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString()); } else { StringBuilder sb = new StringBuilder(); sb.Append("Latest Delta: "); foreach (KeyValuePair <int, long> kv in latestOffsetDictLastValue) { if (latestOffsetDict.ContainsKey(kv.Key)) { sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value); } else { sb.AppendFormat("Latest:{0,-9} ", kv.Value); } } foreach (KeyValuePair <int, long> kv in latestOffsetDict) { if (!latestOffsetDictLastValue.ContainsKey(kv.Key)) { sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value); } } Logger.Info(sb.ToString()); KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString()); } latestOffsetDictLastValue = latestOffsetDict; Thread.Sleep(produceMonitorOptions.IntervalInSeconds * 1000); } } }
internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions) { string topics = dtOptions.Topic; string zookeeper = dtOptions.Zookeeper; int partitionIndex = dtOptions.PartitionIndex; bool includePartitionDetailInfo = true; bool includeOffsetInfo = true; DateTime timestamp = dtOptions.TimestampInUTC; bool dumpToLog = true; bool dumpToConsole = true; string file = dtOptions.File; using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper, ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer)) { zkClient.Connect(); //BrokerID -->Count of as leader SortedDictionary <int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary <int, int>(); //BrokerID -->Count of as replica SortedDictionary <int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary <int, int>(); StringBuilder sbAll = new StringBuilder(); int topicCount = 0; if (string.IsNullOrEmpty(topics)) { List <string> topicList = new List <string>(); string path = "/brokers/topics"; try { IEnumerable <string> ts = zkClient.GetChildren(path); foreach (var p in ts) { topicList.Add(p); } } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { throw new ApplicationException("Please make sure the path exists in zookeeper: " + path, e); } else { throw; } } sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count); foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray()) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } topicCount = topicList.Count; } else if (topics.Contains(",")) { string[] topicArray = topics.Split(new char[] { ',' }); topicCount = topicArray.Length; sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length); foreach (var t in topicArray.OrderBy(r => r).ToArray()) { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); } } else { SortedDictionary <int, long> latestOffsetDict = new SortedDictionary <int, long>(); SortedDictionary <int, long> latestLength = new SortedDictionary <int, long>(); sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics, zookeeper, partitionIndex, includePartitionDetailInfo, includeOffsetInfo, timestamp, parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength)); topicCount = 1; } if (topicCount > 1) { sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n"); sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n"); foreach (KeyValuePair <int, int> kv in parttionBrokerID_LeaderCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n"); sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n"); foreach (KeyValuePair <int, int> kv in parttionBrokerID_ReplicaCountDistrib) { sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value); } } string s = sbAll.ToString(); if (dumpToLog) { Logger.Info(s); } if (dumpToConsole) { Console.WriteLine(s); } if (!string.IsNullOrEmpty(file)) { Console.WriteLine("Will write to {0}", file); using (StreamWriter sw = new StreamWriter(file, false)) { sw.WriteLine(s); } } } }