public BrokerPartitionInfo(ISyncProducerPool syncProducerPool, IDictionary<string, TopicMetadata> cache, IDictionary<string, DateTime> lastUpdateTime, int topicMetaDataRefreshIntervalMS, ZooKeeperClient zkClient)
 {
     this.syncProducerPool = syncProducerPool;
     this.topicPartitionInfo = cache;
     this.topicPartitionInfoLastUpdateTime = lastUpdateTime;
     this.topicMetaDataRefreshIntervalMS = topicMetaDataRefreshIntervalMS;
     this.zkClient = zkClient;
 }
        public SyncProducerPool(ProducerConfiguration config)
        {
            this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>();
            this.Config = config;

            if (config.ZooKeeper != null)
            {
                this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
                                                   ZooKeeperStringSerializer.Serializer);
                this.zkClient.Connect();
            }

            this.AddProducers(config);
        }
        public SyncProducerPool(ProducerConfiguration config, List<ISyncProducer> producers)
        {
            this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>();
            this.Config = config;

            if (config.ZooKeeper != null)
            {
                this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
                                                   ZooKeeperStringSerializer.Serializer);
                this.zkClient.Connect();
            }

            if (producers != null && producers.Any())
            {
                producers.ForEach(x => this.syncProducers.TryAdd(x.Config.BrokerId, new SyncProducerWrapper(x, config.SyncProducerOfOneBroker)));
            }
        }
Esempio n. 4
0
        public void WhenBrokerIsRemovedBrokerTopicsListenerUpdatesBrokersList()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            IDictionary<string, SortedSet<Partition>> mappings;
            IDictionary<int, Broker> brokers;
            string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    brokers = brokerPartitionInfo.GetAllBrokerInfo();
                    mappings =
                        ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                            "topicBrokerPartitions", brokerPartitionInfo);
                }
            }

            Assert.NotNull(brokers);
            Assert.Greater(brokers.Count, 0);
            Assert.NotNull(mappings);
            Assert.Greater(mappings.Count, 0);
            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                WaitUntillIdle(client, 500);
                var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null);
                client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener);
                client.CreatePersistent(brokerPath, true);
                client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
                WaitUntillIdle(client, 500);
                Assert.IsTrue(brokers.ContainsKey(2345));
                client.DeleteRecursive(brokerPath);
                WaitUntillIdle(client, 500);
                Assert.IsFalse(brokers.ContainsKey(2345));
            }
        }
        internal static void DumpConsumerGroupOffsets(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption)
        {
            List<ConsumeGroupMonitorUnit> units = new List<ConsumeGroupMonitorUnit>();
            if (string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumeGroupTopicArray))
            {
                if (!string.IsNullOrEmpty(consumeGroupMonitorOption.ConsumerGroupName))
                    units.Add(new ConsumeGroupMonitorUnit(consumeGroupMonitorOption.File, consumeGroupMonitorOption.ConsumerGroupName, consumeGroupMonitorOption.Topic));
                else
                {
                    danymicAllConsumeGroupTopic = true;
                }
            }
            else
            {
                string[] cgs = consumeGroupMonitorOption.ConsumeGroupTopicArray.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                foreach (var cg in cgs)
                {
                    string[] temp = cg.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);
                    if (temp.Length != 2)
                    {
                        Logger.ErrorFormat("Wrong parameter, Exmaple  G1:t1,G2:t2.  The value:{0} is invalid.", cg);
                    }
                    else
                        units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, temp[0], temp[1]), temp[0], temp[1]));
                }
            }
            long count = 0;
            int cycle = consumeGroupMonitorOption.RefreshConsumeGroupIntervalInSeconds / consumeGroupMonitorOption.IntervalInSeconds;
            using (ZooKeeperClient zkClient = new ZooKeeperClient(consumeGroupMonitorOption.Zookeeper,
                         ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
            {
                zkClient.Connect();
                while (true)
                {
                    DateTime time = DateTime.Now;
                    string dateFolder = string.Format("{0}_{1}_{2}", time.Year, time.Month, time.Day);
                    if (!Directory.Exists(dateFolder))
                        Directory.CreateDirectory(dateFolder);

                    if (danymicAllConsumeGroupTopic && count % cycle == 0)
                    {
                        units = new List<ConsumeGroupMonitorUnit>();
                        RefreshConsumeGroup(consumeGroupMonitorOption, zkClient, units);
                    }

                    int countSuccess = 0;
                    foreach (var unit in units)
                    {
                        unit.subFolder = dateFolder;
                        if (unit.Run(zkClient, consumeGroupMonitorOption.Zookeeper))
                            countSuccess++;
                    }

                    if (countSuccess == units.Count)
                        Logger.InfoFormat("===========All {0} consume group PASS=================", countSuccess);
                    else
                        Logger.ErrorFormat("===========All {0} consume group only {1} succ.  FAIL=================", units.Count, countSuccess);

                    Thread.Sleep(consumeGroupMonitorOption.IntervalInSeconds * 1000);
                    count++;

                }
            }
        }
        internal static SortedDictionary<int, string> GetComsumerGroupOwners(ZooKeeperClient zkClient,
         string topic, string consumerGroupName)
        {
            SortedDictionary<int, string> partitionsOwners = new SortedDictionary<int, string>();

            string path = string.Format("/consumers/{0}/owners/{1}"
                    , consumerGroupName, topic);

            IEnumerable<string> partitions = zkClient.GetChildrenParentMayNotExist(path);
            if (partitions != null)
            {
                foreach (var p in partitions)
                {
                    string fullPatht = string.Format("/consumers/{0}/owners/{1}/{2}"
                    , consumerGroupName, topic, p);
                    string data = zkClient.ReadData<string>(fullPatht, true);
                    partitionsOwners.Add(Convert.ToInt32(p), data);
                }
            }

            return partitionsOwners;
        }
Esempio n. 7
0
        public void WhenChildIsCreatedChilListenerOnParentFires()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            string myPath = "/" + Guid.NewGuid();
            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                WaitUntillIdle(client, 500);
                client.Subscribe("/", this as IZooKeeperChildListener);
                client.CreatePersistent(myPath, true);
                WaitUntillIdle(client, 500);
                client.UnsubscribeAll();
                client.Delete(myPath);
            }

            Assert.AreEqual(1, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperChildChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, "/");
            Assert.Greater(((ZooKeeperChildChangedEventArgs)e).Children.Count, 0);
            Assert.IsTrue(((ZooKeeperChildChangedEventArgs)e).Children.Contains(myPath.Replace("/", string.Empty)));
        }
Esempio n. 8
0
        public void ZooKeeperClientCreateWorkerThreadsOnBeingCreated()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                var eventWorker = ReflectionHelper.GetInstanceField<Thread>("eventWorker", client);
                var zooKeeperWorker = ReflectionHelper.GetInstanceField<Thread>("zooKeeperEventWorker", client);
                Assert.NotNull(eventWorker);
                Assert.NotNull(zooKeeperWorker);
            }
        }
Esempio n. 9
0
        public void ZooKeeperClientCreatesAChildAndGetsChildren()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                string child = Guid.NewGuid().ToString();
                string myPath = "/" + child;
                client.CreatePersistent(myPath, false);
                IList<string> children = client.GetChildren("/", false);
                int countChildren = client.CountChildren("/");
                Assert.Greater(children.Count, 0);
                Assert.AreEqual(children.Count, countChildren);
                Assert.IsTrue(children.Contains(child));
                client.Delete(myPath);
            }
        }
Esempio n. 10
0
        public void WhenZNodeIsDeletedChildAndDataDeletedListenersFire()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            string myPath = "/" + Guid.NewGuid();
            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                client.CreatePersistent(myPath, true);
                WaitUntillIdle(client, 500);
                client.Subscribe(myPath, this as IZooKeeperChildListener);
                client.Subscribe(myPath, this as IZooKeeperDataListener);
                client.Delete(myPath);
                WaitUntillIdle(client, 500);
            }

            Assert.AreEqual(2, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperChildChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, myPath);
            Assert.IsNull(((ZooKeeperChildChangedEventArgs)e).Children);
            e = this.events[1];
            Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperDataChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
            Assert.IsNull(((ZooKeeperDataChangedEventArgs)e).Data);
        }
Esempio n. 11
0
        public void WhenStateChangedToDisconnectedStateListenerFires()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
            {
                client.Subscribe(this);
                client.Connect();
                WaitUntillIdle(client, 500);
                client.Process(new WatchedEvent(KeeperState.Disconnected, EventType.None, null));
                WaitUntillIdle(client, 500);
            }

            Assert.AreEqual(2, this.events.Count);
            ZooKeeperEventArgs e = this.events[1];
            Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.Disconnected);
        }
        internal static void DumpTopicMetadataAndOffset(TopicHelperArguments dtOptions)
        {
            string topics = dtOptions.Topic;
            string zookeeper = dtOptions.Zookeeper;
            int partitionIndex = dtOptions.PartitionIndex;
            bool includePartitionDetailInfo = true;
            bool includeOffsetInfo = true;
            DateTime timestamp = dtOptions.TimestampInUTC;
            bool dumpToLog = true;
            bool dumpToConsole = true;
            string file = dtOptions.File;

            using (ZooKeeperClient zkClient = new ZooKeeperClient(zookeeper,
                            ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
            {
                zkClient.Connect();
                //BrokerID -->Count of as leader
                SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
                //BrokerID -->Count of as replica
                SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();

                StringBuilder sbAll = new StringBuilder();
                int topicCount = 0;
                if (string.IsNullOrEmpty(topics))
                {
                    List<string> topicList = new List<string>();
                    string path = "/brokers/topics";
                    try
                    {
                        IEnumerable<string> ts = zkClient.GetChildren(path);
                        foreach (var p in ts)
                        {
                            topicList.Add(p);
                        }
                    }
                    catch (KeeperException e)
                    {
                        if (e.ErrorCode == KeeperException.Code.NONODE)
                        {
                            throw new ApplicationException("Please make sure the path exists in zookeeper:  " + path, e);
                        }
                        else
                            throw;
                    }

                    sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicList.Count);
                    foreach (var t in topicList.ToArray().OrderBy(r => r).ToArray())
                    {
                        SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
                        SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
                        sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
                        zookeeper,
                        partitionIndex,
                        includePartitionDetailInfo,
                        includeOffsetInfo,
                        timestamp,
                        parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
                    }
                    topicCount = topicList.Count;
                }
                else if (topics.Contains(","))
                {
                    string[] topicArray = topics.Split(new char[] { ',' });
                    topicCount = topicArray.Length;
                    sbAll.AppendFormat("\r\nTotally {0} topics. \r\n\r\n", topicArray.Length);
                    foreach (var t in topicArray.OrderBy(r => r).ToArray())
                    {
                        SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
                        SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
                        sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, t,
                       zookeeper,
                       partitionIndex,
                       includePartitionDetailInfo,
                       includeOffsetInfo,
                       timestamp,
                       parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
                    }
                }
                else
                {
                    SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
                    SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
                    sbAll.Append(DumpTopicMetadataAndOffsetInternal(zkClient, topics,
                        zookeeper,
                        partitionIndex,
                        includePartitionDetailInfo,
                        includeOffsetInfo,
                        timestamp,
                       parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength));
                    topicCount = 1;
                }

                if (topicCount > 1)
                {
                    sbAll.AppendFormat("\r\nBroker as leader distribution=====All topic=======\r\n");
                    sbAll.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
                    foreach (KeyValuePair<int, int> kv in parttionBrokerID_LeaderCountDistrib)
                    {
                        sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
                    }

                    sbAll.AppendFormat("Broker as replica distribution========All topic=====\r\n");
                    sbAll.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
                    foreach (KeyValuePair<int, int> kv in parttionBrokerID_ReplicaCountDistrib)
                    {
                        sbAll.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
                    }
                }

                string s = sbAll.ToString();
                if (dumpToLog)
                {
                    Logger.Info(s);
                }

                if (dumpToConsole)
                {
                    Console.WriteLine(s);
                }

                if (!string.IsNullOrEmpty(file))
                {
                    Console.WriteLine("Will write to {0}", file);
                    using (StreamWriter sw = new StreamWriter(file, false))
                    {
                        sw.WriteLine(s);
                    }
                }
            }
        }
 //{"controller_epoch":4,"leader":2,"version":1,"leader_epoch":5,"isr":[2]}
 internal static ArrayList GetIsr(ZooKeeperClient zkClient, string topic, int partition)
 {
     string data = zkClient.ReadData<string>(string.Format("/brokers/topics/{0}/partitions/{1}/state"
         , topic, partition), true);
     Dictionary<string, object> ctx = new JavaScriptSerializer().Deserialize<Dictionary<string, object>>(data);
     Type ty = ctx["isr"].GetType();
     return (ArrayList)ctx["isr"];
 }
        internal static string DumpTopicMetadataAndOffsetInternal(ZooKeeperClient zkClient, string topic,
           string zookeeper,
           int partitionIndex,
           bool includePartitionDetailInfo,
           bool includeOffsetInfo,
           DateTime timestamp,
           SortedDictionary<int, int> parttionBrokerID_LeaderCountDistribAll,
           SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistribAll,
           SortedDictionary<int, long> latestOffset,
           SortedDictionary<int, long> latestLength)
        {
            StringBuilder sb = new StringBuilder();
            string s = string.Empty;
            //BrokerID -->Count of as leader
            SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
            //BrokerID -->Count of as replica
            SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
            try
            {
                if (string.IsNullOrEmpty(zookeeper))
                {
                    Logger.Error(" zookeeper  should be provided");
                    sb.AppendFormat(DumpTopicError, topic);
                }
                else
                {
                    KafkaSimpleManagerConfiguration config = new KafkaSimpleManagerConfiguration()
                    {
                        Zookeeper = zookeeper
                    };
                    config.Verify();
                    Dictionary<int, int[]> detailDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(zkClient, topic);
                    using (KafkaSimpleManager<int, Message> kafkaSimpleManager = new KafkaSimpleManager<int, Message>(config))
                    {
                        TopicMetadata topicMetadata = kafkaSimpleManager.RefreshMetadata(KafkaNETExampleConstants.DefaultVersionId, ClientID, TopicMetadataRequestID++, topic, true);

                        int partitionCount = topicMetadata.PartitionsMetadata.Count();
                        sb.AppendFormat("Topic:{0}\tPartitionCount:{1}\t", topic, partitionCount);

                        int replicationFactor = Enumerable.Count<Broker>(topicMetadata.PartitionsMetadata.First().Replicas);
                        sb.AppendFormat("ReplicationFactor:{0}\t", replicationFactor);

                        //TODO:  compare detailDataInZookeeper and check which one missed.
                        StringBuilder sbDetail = new StringBuilder();
                        if (includePartitionDetailInfo)
                        {
                            long sumEndOffset = 0;
                            long sumLength = 0;
                            foreach (PartitionMetadata p in topicMetadata.PartitionsMetadata.OrderBy(r => r.PartitionId).ToList())
                            {
                                int[] replicaInZookeeper = null;
                                if (detailDataInZookeeper.ContainsKey(p.PartitionId))
                                {
                                    replicaInZookeeper = detailDataInZookeeper[p.PartitionId];
                                    detailDataInZookeeper.Remove(p.PartitionId);
                                }

                                #region One partition
                                long earliest = 0;
                                long latest = 0;
                                if (partitionIndex == -1 || p.PartitionId == partitionIndex)
                                {
                                    //sbDetail.AppendFormat("\tTopic:{0}", topic);
                                    sbDetail.AppendFormat("\tPartition:{0}", p.PartitionId);
                                    if (p.Leader != null)
                                    {
                                        sbDetail.AppendFormat("\tLeader:{0}", KafkaConsoleUtil.GetBrokerIDAndIP(p.Leader.Id));

                                        if (parttionBrokerID_LeaderCountDistrib.ContainsKey(p.Leader.Id))
                                            parttionBrokerID_LeaderCountDistrib[p.Leader.Id]++;
                                        else
                                            parttionBrokerID_LeaderCountDistrib.Add(p.Leader.Id, 1);

                                        if (parttionBrokerID_LeaderCountDistribAll.ContainsKey(p.Leader.Id))
                                            parttionBrokerID_LeaderCountDistribAll[p.Leader.Id]++;
                                        else
                                            parttionBrokerID_LeaderCountDistribAll.Add(p.Leader.Id, 1);
                                    }
                                    else
                                        sbDetail.AppendFormat("\tLeader:NoLeader!");

                                    sbDetail.AppendFormat("\tReplicas:{0}", string.Join(",", p.Replicas.Select(r => KafkaConsoleUtil.GetBrokerIDAndIP(r.Id)).ToArray()));
                                    foreach (Broker b in p.Replicas)
                                    {
                                        if (parttionBrokerID_ReplicaCountDistrib.ContainsKey(b.Id))
                                            parttionBrokerID_ReplicaCountDistrib[b.Id]++;
                                        else
                                            parttionBrokerID_ReplicaCountDistrib.Add(b.Id, 1);

                                        if (parttionBrokerID_ReplicaCountDistribAll.ContainsKey(b.Id))
                                            parttionBrokerID_ReplicaCountDistribAll[b.Id]++;
                                        else
                                            parttionBrokerID_ReplicaCountDistribAll.Add(b.Id, 1);
                                    }

                                    //sbDetail.AppendFormat("\tIsr:{0}", string.Join(",", p.Isr.Select(r => r.Id).ToArray()));
                                    ArrayList isrs = GetIsr(zkClient, topic, p.PartitionId);
                                    sbDetail.AppendFormat("\tIsr:{0}", string.Join(",", isrs.ToArray().Select(r => KafkaConsoleUtil.GetBrokerIDAndIP(Convert.ToInt32(r)))));
                                    //TODO: add missed replica

                                    #region Offset
                                    if (includeOffsetInfo)
                                    {
                                        try
                                        {
                                            kafkaSimpleManager.RefreshAndGetOffset(KafkaNETExampleConstants.DefaultVersionId, ClientID, TopicMetadataRequestID++, topic, p.PartitionId, true, out earliest, out latest);
                                            sumEndOffset += latest;
                                            sumLength += (latest - earliest);
                                            sbDetail.AppendFormat("\tlength:{2}\tearliest:{0}\tlatest:{1}"
                                                , earliest
                                                , latest
                                                , (latest - earliest) == 0 ? "(empty)" : (latest - earliest).ToString());
                                            sbDetail.AppendFormat("\r\n");

                                            latestOffset.Add(p.PartitionId, latest);
                                            latestLength.Add(p.PartitionId, latest - earliest);
                                        }
                                        catch (NoLeaderForPartitionException e)
                                        {
                                            sbDetail.AppendFormat(" ERROR:{0}\r\n", e.Message);
                                        }
                                        catch (UnableToConnectToHostException e)
                                        {
                                            sbDetail.AppendFormat(" ERROR:{0}\r\n", e.Message);
                                        }

                                        if (timestamp != DateTime.MinValue)
                                        {

                                            long timestampLong = KafkaClientHelperUtils.ToUnixTimestampMillis(timestamp);
                                            try
                                            {
                                                long timeStampOffset = kafkaSimpleManager.RefreshAndGetOffsetByTimeStamp(KafkaNETExampleConstants.DefaultVersionId, ClientID, TopicMetadataRequestID++, topic, p.PartitionId, timestamp);
                                                sbDetail.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tUnixTimestamp:{2}\t"
                                               , timeStampOffset
                                               , KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s")
                                               , timestampLong);
                                                sbDetail.AppendFormat("\r\n");
                                            }
                                            catch (TimeStampTooSmallException)
                                            {
                                                sbDetail.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tUnixTimestamp:{2}\t"
                                                 , "NA since no data before the time you specified, please retry with a bigger value."
                                                 , KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s")
                                                 , timestampLong);
                                                sbDetail.AppendFormat("\r\n");
                                            }

                                        }
                                    }
                                    #endregion
                                }
                                #endregion
                            }
                            if (includeOffsetInfo)
                            {
                                sb.AppendFormat("SumeEndOffset:{0:0,0}  SumLength:{1:0,0}\r\n", sumEndOffset, sumLength);
                            }
                            else
                            {
                                sb.AppendFormat("\r\n");
                            }

                            if (detailDataInZookeeper.Any())
                            {
                                foreach (KeyValuePair<int, int[]> kv in detailDataInZookeeper)
                                {
                                    sb.AppendFormat("=ERROR=MISSED partition= {0}  Replicas {1} ", kv.Key, string.Join(",", kv.Value.Select(r => r.ToString()).ToArray()));
                                }
                            }
                        }

                        sb.Append(sbDetail.ToString());
                        sb.AppendFormat("\tBroker as leader distribution======={0}=======\r\n", topic);
                        sb.AppendFormat("\r\tBrokerID\tLeadPartition count\r\n");
                        foreach (KeyValuePair<int, int> kv in parttionBrokerID_LeaderCountDistrib)
                        {
                            sb.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
                        }

                        sb.AppendFormat("\tBroker as replica distribution========={0}=====\r\n", topic);
                        sb.AppendFormat("\r\tBrokerID\tReplication count count\r\n");
                        foreach (KeyValuePair<int, int> kv in parttionBrokerID_ReplicaCountDistrib)
                        {
                            sb.AppendFormat("\t\t{0}\t{1}\r\n", KafkaConsoleUtil.GetBrokerIDAndIP(kv.Key), kv.Value);
                        }

                        sb.AppendFormat("\r\n");
                    }
                }

                s = sb.ToString();
            }
            catch (NoBrokerForTopicException e)
            {
                sb.AppendFormat("\r\nTopic:{0}\t ==NoBrokerForTopicException:{1}!!!== \r\n", topic, e.Message);
                s = sb.ToString();
            }
            catch (UnableToConnectToHostException e)
            {
                sb.AppendFormat("\r\nTopic:{0}\t ==UnableToConnectToHostException:{1}!!!== \r\n", topic, e.Message);
                s = sb.ToString();
            }
            catch (Exception ex)
            {
                Logger.ErrorFormat("Dump topic got exception:{0}\r\ninput parameter:Topic:{1}\tZookeeper:{2}\tKafka:{3}\tPartionIndex:{4}\tincludePartitionDetailInfo:{5}\tincludeOffsetInfo:{6}\ttimestamp:{7}\r\nPartial result:{8}"
                     , ExceptionUtil.GetExceptionDetailInfo(ex),
                     topic,
                     zookeeper,
                     string.Empty,
                     partitionIndex,
                     includePartitionDetailInfo,
                     includeOffsetInfo,
                     timestamp,
                     s);
            }

            return s;
        }
Esempio n. 15
0
        //[zk: localhost(CONNECTED) 12] get /brokers/topics/mvlogs
        //{"version":1,"partitions":{"1":[3,2],"0":[2,3]}}
        public static Dictionary<int, int[]> GetTopicMetadataInzookeeper(ZooKeeperClient zkClient, string topic)
        {
            Dictionary<int, int[]> treturn = new Dictionary<int, int[]>();

            try
            {
                string data = zkClient.ReadData<string>(string.Format("/brokers/topics/{0}", topic), true);
                Dictionary<string, object> ctx = new JavaScriptSerializer().Deserialize<Dictionary<string, object>>(data);
                Type ty = ctx["partitions"].GetType();
                //Logger.InfoFormat("The type for partitions :{0}", ty.FullName);
                Dictionary<string, object> tpartitons = (Dictionary<string, object>)ctx["partitions"];

                foreach (KeyValuePair<string, object> kv in tpartitons)
                {
                    int partitionID = Convert.ToInt32(kv.Key);
                    //Logger.InfoFormat("The type for partitions value :{0}", kv.Value.GetType().FullName);
                    ArrayList rep = (ArrayList)kv.Value;
                    int[] partitionReplicas = new int[rep.Count];

                    for (int i = 0; i < rep.Count; i++)
                        partitionReplicas[i] = Convert.ToInt32(rep[i]);
                    treturn.Add(partitionID, partitionReplicas);
                }
                Logger.InfoFormat("Get topic data directly from zookeeper Topic:{0} Data:{1} Partition count:{2}", topic, data, treturn.Count);
            }
            catch (Exception ex)
            {
                Logger.Error("Failed to get topic " + topic + " data directly from zookeeper: " + ex.FormatException());
            }

            return treturn;
        }
Esempio n. 16
0
        public void WhenStateChangedToConnectedStateListenerFires()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Subscribe(this);
                client.Connect();
                WaitUntillIdle(client, 500);
            }

            Assert.AreEqual(1, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
        }
Esempio n. 17
0
        public void WhenStateChangedToExpiredStateAndSessionListenersFire()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Subscribe(this);
                client.Connect();
                WaitUntillIdle(client, 500);
                client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
                WaitUntillIdle(client, 3000);
            }

            Assert.AreEqual(4, this.events.Count);
            ZooKeeperEventArgs e = this.events[1];
            Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.Expired);
            e = this.events[2];
            Assert.AreEqual(ZooKeeperEventTypes.SessionCreated, e.Type);
            Assert.IsInstanceOf<ZooKeeperSessionCreatedEventArgs>(e);
            e = this.events[3];
            Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
        }
Esempio n. 18
0
        public void WhenNewBrokerInTopicIsAddedZKBrokerPartitionInfoUpdatesMappings()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IDictionary<string, SortedSet<Partition>> mappings;
            IDictionary<int, Broker> brokers;
            string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
            string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic;
            string topicBrokerPath = topicPath + "/" + 2345;
            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    brokers = brokerPartitionInfo.GetAllBrokerInfo();
                    mappings =
                        ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                            "topicBrokerPartitions", brokerPartitionInfo);
                    client.CreatePersistent(brokerPath, true);
                    client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
                    client.CreatePersistent(topicPath, true);
                    WaitUntillIdle(client, 500);
                    Assert.IsTrue(brokers.ContainsKey(2345));
                    Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic));
                    client.CreatePersistent(topicBrokerPath, true);
                    client.WriteData(topicBrokerPath, 5);
                    WaitUntillIdle(client, 500);
                    client.UnsubscribeAll();
                    client.DeleteRecursive(brokerPath);
                    client.DeleteRecursive(topicPath);
                }
            }

            Assert.NotNull(brokers);
            Assert.Greater(brokers.Count, 0);
            Assert.NotNull(mappings);
            Assert.Greater(mappings.Count, 0);
            Assert.IsTrue(brokers.ContainsKey(2345));
            Assert.IsTrue(mappings.Keys.Contains(CurrentTestTopic));
            Assert.AreEqual(5, mappings[CurrentTestTopic].Count);
        }
Esempio n. 19
0
        public void ZooKeeperClientChecksIfPathExists()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                Assert.IsTrue(client.Exists(ZooKeeperClient.DefaultBrokerTopicsPath, false));
            }
        }
Esempio n. 20
0
        public void WhenNewTopicIsAddedZKBrokerPartitionInfoUpdatesMappings()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IDictionary<string, SortedSet<Partition>> mappings;
            string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic;

            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    mappings =
                        ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                            "topicBrokerPartitions", brokerPartitionInfo);
                    client.CreatePersistent(topicPath, true);
                    WaitUntillIdle(client, 500);
                    client.UnsubscribeAll();
                    client.DeleteRecursive(topicPath);
                }
            }

            Assert.NotNull(mappings);
            Assert.Greater(mappings.Count, 0);
            Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic));
        }
Esempio n. 21
0
        public void ZooKeeperClientCreatesANewPathAndDeletesIt()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                string myPath = "/" + Guid.NewGuid();
                client.CreatePersistent(myPath, false);
                Assert.IsTrue(client.Exists(myPath));
                client.Delete(myPath);
                Assert.IsFalse(client.Exists(myPath));
            }
        }
Esempio n. 22
0
        public void WhenSessionIsExpiredListenerRecreatesEphemeralNodes()
        {
            {
                var producerConfig = new ProducerConfig(clientConfig);
                IDictionary<string, SortedSet<Partition>> mappings;
                IDictionary<int, Broker> brokers;
                IDictionary<string, SortedSet<Partition>> mappings2;
                IDictionary<int, Broker> brokers2;
                using (IZooKeeperClient client = new ZooKeeperClient(
                    producerConfig.ZkConnect,
                    producerConfig.ZkSessionTimeoutMs,
                    ZooKeeperStringSerializer.Serializer))
                {
                    using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                    {
                        brokers = brokerPartitionInfo.GetAllBrokerInfo();
                        mappings =
                            ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                                "topicBrokerPartitions", brokerPartitionInfo);
                        Assert.NotNull(brokers);
                        Assert.Greater(brokers.Count, 0);
                        Assert.NotNull(mappings);
                        Assert.Greater(mappings.Count, 0);
                        client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
                        WaitUntillIdle(client, 3000);
                        brokers2 = brokerPartitionInfo.GetAllBrokerInfo();
                        mappings2 =
                            ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                                "topicBrokerPartitions", brokerPartitionInfo);
                    }
                }

                Assert.NotNull(brokers2);
                Assert.Greater(brokers2.Count, 0);
                Assert.NotNull(mappings2);
                Assert.Greater(mappings2.Count, 0);
                Assert.AreEqual(brokers.Count, brokers2.Count);
                Assert.AreEqual(mappings.Count, mappings2.Count);
            }
        }
Esempio n. 23
0
        public void ZooKeeperClientFailsWhenCreatedWithWrongConnectionInfo()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client = new ZooKeeperClient(
                "random text",
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                Assert.Throws<FormatException>(client.Connect);
            }
        }
Esempio n. 24
0
        public void WhenBrokerIsRemovedZKBrokerPartitionInfoUpdatesBrokersList()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IDictionary<int, Broker> brokers;
            string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    brokers = brokerPartitionInfo.GetAllBrokerInfo();
                    client.CreatePersistent(brokerPath, true);
                    client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
                    WaitUntillIdle(client, 500);
                    Assert.NotNull(brokers);
                    Assert.Greater(brokers.Count, 0);
                    Assert.IsTrue(brokers.ContainsKey(2345));
                    client.DeleteRecursive(brokerPath);
                    WaitUntillIdle(client, 500);
                }
            }

            Assert.NotNull(brokers);
            Assert.Greater(brokers.Count, 0);
            Assert.IsFalse(brokers.ContainsKey(2345));
        }
        private static void RefreshConsumeGroup(ConsumeGroupMonitorHelperOptions consumeGroupMonitorOption, ZooKeeperClient zkClient, List<ConsumeGroupMonitorUnit> units)
        {
            Logger.Info("Will refresh all consume group from zk ...");
            string path = "/consumers";
            //TODO: add /users
            IEnumerable<string> groups = zkClient.GetChildren(path);
            foreach (var g in groups)
            {
                Logger.InfoFormat("Will check group {0}", g);
                string topicpath = string.Format("/consumers/{0}/owners", g);
                IEnumerable<string> topics = zkClient.GetChildren(topicpath);
                foreach (var t in topics)
                {
                    Logger.InfoFormat("Will check topic {0}", t);
                    units.Add(new ConsumeGroupMonitorUnit(string.Format("{0}_{1}_{2}.txt", consumeGroupMonitorOption.File, g, t), g, t));
                }
            }

            Logger.InfoFormat("Finsh refresh all consume group from zk ...{0}", units.Count);
        }
Esempio n. 26
0
        public void WhenClientWillNotConnectWithinGivenTimeThrows()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            using (IZooKeeperClient client =
                new ZooKeeperClient(
                    prodConfig.ZooKeeper.ZkConnect,
                    prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                    ZooKeeperStringSerializer.Serializer,
                    1))
            {
                client.Connect();
            }
        }
        internal bool Run(ZooKeeperClient zkClient, string zookeeper)
        {
            bool success = true;
            SortedDictionary<int, long> latestCommited = GetComsumerGroupOffsets(zkClient, this.topic, this.group);
            SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
            SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
            //BrokerID -->Count of as replica
            SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
            SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
            //Owner
            SortedDictionary<int, string> latestOwners = GetComsumerGroupOwners(zkClient, this.topic, this.group);

            TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, this.topic,
            zookeeper,
            -1,
            true,
            true,
            DateTime.MinValue,
            parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength);

            if (latestOffsetDictLastValue == null)
            {
                StringBuilder sb = new StringBuilder();
                sb.AppendFormat("====Partitions====\r\n");
                foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                {
                    sb.AppendFormat("{0,-9} ", kv.Key);
                }

                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                sb = new StringBuilder();
                sb.AppendFormat("====LatestOffset====\r\n");
                foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                {
                    sb.AppendFormat("{0,-9} ", kv.Value);
                }

                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                sb = new StringBuilder();
                sb.AppendFormat("====ConsumedOffset====\r\n");
                foreach (KeyValuePair<int, long> kv in latestCommited)
                {
                    sb.AppendFormat("{0,-9} ", kv.Value);
                }

                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                sb = new StringBuilder();
                sb.AppendFormat("===Latest-Earliest:  Initial value====\r\n");
                foreach (KeyValuePair<int, long> kv in latestLength)
                {
                    sb.AppendFormat("{0,-9} ", kv.Value);
                }

                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                sb = new StringBuilder();
                sb.AppendFormat("====Latest-Commited:  Initial Value====\r\n");
                foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                {
                    if (latestCommited.ContainsKey(kv.Key))
                        sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]);
                    else
                        sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value);
                }

                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                sb = new StringBuilder();
                if (latestOffsetDict.Count == latestOwners.Count)
                {
                    sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count);
                }
                else
                {
                    sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count);
                }

                foreach (var ownerByOwnerName in (from o in latestOwners
                                                  group o by o.Value into g
                                                  select new
                                                  {
                                                      owner = g.Key,
                                                      partitions = g.ToArray()
                                                  }).OrderBy(r => r.owner))
                {
                    sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length);
                    for (int k = 0; k < ownerByOwnerName.partitions.Length; k++)
                    {
                        if (k == 0)
                            sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]);
                        else
                            sb.AppendFormat(",   {0}", ownerByOwnerName.partitions[k]);
                    }

                    sb.AppendFormat("\r\n");
                }
                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString());

            }
            else
            {
                //Length
                StringBuilder sb = new StringBuilder();
                sb.Append("Latest-Earliest: ");
                foreach (KeyValuePair<int, long> kv in latestLength)
                {
                    sb.AppendFormat("{0,-9} ", kv.Value);
                }
                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                //Latest Delta
                sb = new StringBuilder();
                long latestDelta = 0;
                long aggregateLatestDelta = 0;
                foreach (KeyValuePair<int, long> kv in latestOffsetDictLastValue)
                {
                    if (latestOffsetDict.ContainsKey(kv.Key))
                    {
                        sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value);
                        latestDelta += latestOffsetDict[kv.Key] - kv.Value;
                    }
                    else
                    {
                        sb.AppendFormat("Latest:{0,-9} ", kv.Value);
                    }

                    if (latestOffsetDictFirstValue.ContainsKey(kv.Key))
                    {
                        aggregateLatestDelta += kv.Value - latestOffsetDictFirstValue[kv.Key];
                    }
                }

                foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                {
                    if (!latestOffsetDictLastValue.ContainsKey(kv.Key))
                        sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value);
                }

                sb.Insert(0, string.Format("Latest Delta: {0}", latestDelta));
                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                //Commited Delta
                sb = new StringBuilder();
                long latestDeltaCommited = 0;
                long aggregateLatestCommite = 0;
                foreach (KeyValuePair<int, long> kv in latestCommitedDictLastValue)
                {
                    if (latestCommited.ContainsKey(kv.Key))
                    {
                        sb.AppendFormat("{0,-9} ", latestCommited[kv.Key] - kv.Value);
                        latestDeltaCommited += latestCommited[kv.Key] - kv.Value;
                    }
                    else
                    {
                        sb.AppendFormat("Commited:{0,-9} ", kv.Value);
                    }
                    if (latestCommitedDictFirstValue.ContainsKey(kv.Key))
                    {
                        aggregateLatestCommite += kv.Value - latestCommitedDictFirstValue[kv.Key];
                    }
                }

                foreach (KeyValuePair<int, long> kv in latestCommited)
                {
                    if (!latestCommitedDictLastValue.ContainsKey(kv.Key))
                        sb.AppendFormat("NewCommited:{0}-{1,-9} ", kv.Key, kv.Value);
                }

                sb.Insert(0, string.Format("Commited Delta: {0}", latestDeltaCommited));
                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                //Gap
                sb = new StringBuilder();
                sb.AppendFormat("Latest-Commited: {0}= ", latestOffsetDict.Count);
                foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                {
                    if (latestCommited.ContainsKey(kv.Key))
                        sb.AppendFormat("{0,-9} ", kv.Value - latestCommited[kv.Key]);
                    else
                        sb.AppendFormat("NotComiited:{0}-{1,-9} ", kv.Key, kv.Value);
                }

                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());

                //Owner

                sb = new StringBuilder();
                if (latestOffsetDict.Count == latestOwners.Count)
                {
                    sb.AppendFormat("====Owners== all {0} partitions has owner. ==\r\n", latestOwners.Count);
                }
                else
                {
                    sb.AppendFormat("====Owners ERROR. partitoins: {0} partition has owner: {1} ====\r\n", latestOffsetDict.Count, latestOwners.Count);
                    success = false;
                }

                foreach (var ownerByOwnerName in (from o in latestOwners
                                                  group o by o.Value into g
                                                  select new
                                                  {
                                                      owner = g.Key,
                                                      partitions = g.ToArray()
                                                  }).OrderBy(r => r.owner))
                {
                    sb.AppendFormat("{0}:\t{1}\t", ownerByOwnerName.owner, ownerByOwnerName.partitions.Length);
                    for (int k = 0; k < ownerByOwnerName.partitions.Length; k++)
                    {
                        if (k == 0)
                            sb.AppendFormat("{0}", ownerByOwnerName.partitions[k]);
                        else
                            sb.AppendFormat(",   {0}", ownerByOwnerName.partitions[k]);
                    }
                    sb.AppendFormat("\r\n");
                }

                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendToFile(GetFile(), sb.ToString());
                sb = new StringBuilder();
                sb.AppendFormat("In last {0:0.0} seconds.  Totally latest offset change:{1}   Totally commited offset change:{2} . Percentage:{3:P2} Time:{4}\r\n"
                   , (DateTime.UtcNow - startTime).TotalSeconds, aggregateLatestDelta, aggregateLatestCommite, aggregateLatestCommite * 1.0 / aggregateLatestDelta, DateTime.Now);
                Logger.Info(sb.ToString());
                KafkaNetLibraryExample.AppendLineToFile(GetFile(), sb.ToString());
            }

            previousOwners = latestOwners;
            latestOffsetDictLastValue = latestOffsetDict;
            latestCommitedDictLastValue = latestCommited;
            if (latestOffsetDictFirstValue == null)
                latestOffsetDictFirstValue = latestOffsetDict;
            if (latestCommitedDictFirstValue == null)
                latestCommitedDictFirstValue = latestCommited;
            return success;
        }
Esempio n. 28
0
        public void WhenDataChangedDataListenerFires()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            string myPath = "/" + Guid.NewGuid();
            string sourceData = "my test data";
            string resultData;
            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                client.CreatePersistent(myPath, true);
                WaitUntillIdle(client, 500);
                client.Subscribe(myPath, this as IZooKeeperDataListener);
                client.Subscribe(myPath, this as IZooKeeperChildListener);
                client.WriteData(myPath, sourceData);
                WaitUntillIdle(client, 500);
                client.UnsubscribeAll();
                resultData = client.ReadData<string>(myPath);
                client.Delete(myPath);
            }

            Assert.IsTrue(!string.IsNullOrEmpty(resultData));
            Assert.AreEqual(sourceData, resultData);
            Assert.AreEqual(1, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperDataChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
            Assert.IsNotNull(((ZooKeeperDataChangedEventArgs)e).Data);
            Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Data, sourceData);
        }
        internal static void Run(ProduceMonitorHelperOptions produceMonitorOptions)
        {
            using (ZooKeeperClient zkClient = new ZooKeeperClient(produceMonitorOptions.Zookeeper,
                         ZooKeeperConfiguration.DefaultSessionTimeout, ZooKeeperStringSerializer.Serializer))
            {
                zkClient.Connect();
                while (true)
                {
                    SortedDictionary<int, long> latestOffsetDict = new SortedDictionary<int, long>();
                    SortedDictionary<int, int> parttionBrokerID_LeaderCountDistrib = new SortedDictionary<int, int>();
                    //BrokerID -->Count of as replica
                    SortedDictionary<int, int> parttionBrokerID_ReplicaCountDistrib = new SortedDictionary<int, int>();
                    SortedDictionary<int, long> latestLength = new SortedDictionary<int, long>();
                    TopicHelper.DumpTopicMetadataAndOffsetInternal(zkClient, produceMonitorOptions.Topic,
                    produceMonitorOptions.Zookeeper,
                    -1,
                    true,
                    true,
                    DateTime.MinValue,
                    parttionBrokerID_LeaderCountDistrib, parttionBrokerID_ReplicaCountDistrib, latestOffsetDict, latestLength);
                    if (latestOffsetDictLastValue == null)
                    {
                        StringBuilder sb = new StringBuilder();
                        sb.AppendFormat("====Partitions====\r\n");
                        foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                        {
                            sb.AppendFormat("{0,-9} ", kv.Key);
                        }

                        Logger.Info(sb.ToString());
                        KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
                        sb = new StringBuilder();
                        sb.AppendFormat("====LatestOffset====\r\n");
                        foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                        {
                            sb.AppendFormat("{0,-9} ", kv.Value);
                        }

                        Logger.Info(sb.ToString());
                        KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
                    }
                    else
                    {
                        StringBuilder sb = new StringBuilder();
                        sb.Append("Latest Delta: ");
                        foreach (KeyValuePair<int, long> kv in latestOffsetDictLastValue)
                        {
                            if (latestOffsetDict.ContainsKey(kv.Key))
                            {
                                sb.AppendFormat("{0,-9} ", latestOffsetDict[kv.Key] - kv.Value);
                            }
                            else
                            {
                                sb.AppendFormat("Latest:{0,-9} ", kv.Value);
                            }
                        }

                        foreach (KeyValuePair<int, long> kv in latestOffsetDict)
                        {
                            if (!latestOffsetDictLastValue.ContainsKey(kv.Key))
                                sb.AppendFormat("NewLatest:{0}-{1,-9} ", kv.Key, kv.Value);
                        }

                        Logger.Info(sb.ToString());
                        KafkaNetLibraryExample.AppendLineToFile(produceMonitorOptions.File, sb.ToString());
                    }

                    latestOffsetDictLastValue = latestOffsetDict;
                    Thread.Sleep(produceMonitorOptions.IntervalInSeconds * 1000);
                }
            }
        }
Esempio n. 30
0
        public void WhenSessionExpiredClientReconnects()
        {
            var prodConfig = this.ZooKeeperBasedSyncProdConfig;

            IZooKeeperConnection conn1;
            IZooKeeperConnection conn2;
            using (IZooKeeperClient client = new ZooKeeperClient(
                prodConfig.ZooKeeper.ZkConnect,
                prodConfig.ZooKeeper.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                conn1 = ReflectionHelper.GetInstanceField<ZooKeeperConnection>("connection", client);
                client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
                WaitUntillIdle(client, 1000);
                conn2 = ReflectionHelper.GetInstanceField<ZooKeeperConnection>("connection", client);
            }

            Assert.AreNotEqual(conn1, conn2);
        }