public void ShouldHandleEvents()
 {
     var partitioner = new Mock<IPartitioner<string>>();
     var config = new ProducerConfiguration(new List<BrokerConfiguration>());
     var pool = new Mock<ISyncProducerPool>();
     var producer = new Mock<ISyncProducer>();
     var partitionMetadatas = new List<PartitionMetadata>()
     {
         new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty<Broker>(),
             Enumerable.Empty<Broker>())
     };
     var metadatas = new List<TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.NoError) };
     producer.SetupGet(p => p.Config)
             .Returns(
                 () =>
                     new SyncProducerConfiguration(new ProducerConfiguration(new List<BrokerConfiguration>()), 0,
                         "host1", 1234));
     producer.Setup(p => p.Send(It.IsAny<TopicMetadataRequest>())).Returns(() => metadatas);
     var statuses = new Dictionary<TopicAndPartition, ProducerResponseStatus>();
     statuses[new TopicAndPartition("test", 0)] = new ProducerResponseStatus();
     producer.Setup(p => p.Send(It.IsAny<ProducerRequest>()))
             .Returns(
                 () =>
                     new ProducerResponse(1, statuses));
     pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List<ISyncProducer>() { producer.Object });
     pool.Setup(p => p.GetProducer(It.IsAny<int>())).Returns(() => producer.Object);
     var mockPartitionInfo = new Mock<IBrokerPartitionInfo>();
     mockPartitionInfo.Setup(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test"))
                      .Returns(() =>
                      {
                          var partition = new Partition("test", 0);
                          var replica = new Replica(0, "test");
                          partition.Leader = replica;
                          return new List<Partition>() { partition };
                      });
     var handler = new DefaultCallbackHandler<string, Message>(config, partitioner.Object, new DefaultEncoder(), mockPartitionInfo.Object, pool.Object);
     handler.Handle(new List<ProducerData<string, Message>>() { new ProducerData<string, Message>("test", new Message(new byte[100])) });
     pool.Verify(p => p.GetProducer(0));
 }
Пример #2
0
 /// <summary>
 /// Initializes a new instance of the <see cref="PartitionTopicInfo"/> class.
 /// </summary>
 /// <param name="topic">
 /// The topic.
 /// </param>
 /// <param name="brokerId">
 /// The broker ID.
 /// </param>
 /// <param name="partition">
 /// The broker's partition.
 /// </param>
 /// <param name="chunkQueue">
 /// The chunk queue.
 /// </param>
 /// <param name="consumedOffset">
 /// The consumed offset value.
 /// </param>
 /// <param name="fetchedOffset">
 /// The fetched offset value.
 /// </param>
 /// <param name="fetchSize">
 /// The fetch size.
 /// </param>
 public PartitionTopicInfo(
     string topic,
     int brokerId,
     Partition partition,
     BlockingCollection<FetchedDataChunk> chunkQueue,
     long consumedOffset,
     long fetchedOffset,
     int fetchSize)
 {
     this.Topic = topic;
     this.Partition = partition;
     this.chunkQueue = chunkQueue;
     this.BrokerId = brokerId;
     this.consumedOffset = consumedOffset;
     this.fetchedOffset = fetchedOffset;
     this.FetchSize = fetchSize;
     if (Logger.IsDebugEnabled)
     {
         Logger.DebugFormat(
             CultureInfo.CurrentCulture, "initial consumer offset of {0} is {1}", this, consumedOffset);
         Logger.DebugFormat(
             CultureInfo.CurrentCulture, "initial fetch offset of {0} is {1}", this, fetchedOffset);
     }
 }
Пример #3
0
        /// <summary>
        /// Initializes the topic - broker's partitions mappings.
        /// </summary>
        private void InitializeTopicBrokerPartitions()
        {
            if (this.topicBrokerPartitions != null)
            {
                return;
            }

            this.topicBrokerPartitions = new Dictionary<string, SortedSet<Partition>>();
            this.zkclient.MakeSurePersistentPathExists(ZooKeeperClient.DefaultBrokerTopicsPath);
            IList<string> topics = this.zkclient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerTopicsPath);
            foreach (string topic in topics)
            {
                string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
                IList<string> brokersPerTopic = this.zkclient.GetChildrenParentMayNotExist(brokerTopicPath);
                var brokerPartitions = new SortedDictionary<int, int>();
                foreach (string brokerId in brokersPerTopic)
                {
                    string path = brokerTopicPath + "/" + brokerId;
                    var numPartitionsPerBrokerAndTopic = this.zkclient.ReadData<string>(path);
                    brokerPartitions.Add(int.Parse(brokerId, CultureInfo.InvariantCulture), int.Parse(numPartitionsPerBrokerAndTopic, CultureInfo.CurrentCulture));
                }

                var brokerParts = new SortedSet<Partition>();
                foreach (var brokerPartition in brokerPartitions)
                {
                    for (int i = 0; i < brokerPartition.Value; i++)
                    {
                        var bidPid = new Partition(brokerPartition.Key, i);
                        brokerParts.Add(bidPid);
                    }
                }

                this.topicBrokerPartitions.Add(topic, brokerParts);
            }
        }
Пример #4
0
        private long ResetConsumerOffsets(string topic, Partition partition)
        {
            long offset;
            switch (this.config.AutoOffsetReset)
            {
                case OffsetRequest.SmallestTime:
                    offset = OffsetRequest.EarliestTime;
                    break;
                case OffsetRequest.LargestTime:
                    offset = OffsetRequest.LatestTime;
                    break;
                default:
                    return -1;
            }

            var request = new OffsetRequest(topic, partition.PartId, offset, 1);
            var offsets = this.simpleConsumer.GetOffsetsBefore(request);
            var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic);
            Logger.InfoFormat(CultureInfo.CurrentCulture, "updating partition {0} with {1} offset {2}", partition.Name, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsets[0]);
            ZkUtils.UpdatePersistentPath(this.zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Name, offsets[0].ToString());

            return offsets[0];
        }
Пример #5
0
        /// <summary>
        /// Generate the updated mapping of (brokerId, numPartitions) for the new list of brokers
        /// registered under some topic.
        /// </summary>
        /// <param name="topic">The path of the topic under which the brokers have changed..</param>
        /// <param name="childs">The list of changed brokers.</param>
        private void ProcessNewBrokerInExistingTopic(string topic, IEnumerable<string> childs)
        {
            if (this.actualBrokerTopicsPartitionsMap.ContainsKey(topic))
            {
                Logger.Debug("Old list of brokers -> " + this.oldBrokerTopicsPartitionsMap[topic].ToMultiString(x => x.BrokerId.ToString(), ","));
            }

            var updatedBrokers = new SortedSet<int>(childs.Select(x => int.Parse(x, CultureInfo.InvariantCulture)));
            string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
            var sortedBrokerPartitions = new SortedDictionary<int, int>();
            foreach (var bid in updatedBrokers)
            {
                var num = this.zkclient.ReadData<string>(brokerTopicPath + "/" + bid);
                sortedBrokerPartitions.Add(bid, int.Parse(num, CultureInfo.InvariantCulture));
            }

            var updatedBrokerParts = new SortedSet<Partition>();
            foreach (var bp in sortedBrokerPartitions)
            {
                for (int i = 0; i < bp.Value; i++)
                {
                    var bidPid = new Partition(bp.Key, i);
                    updatedBrokerParts.Add(bidPid);
                }
            }

            Logger.Debug("Currently registered list of brokers for topic " + topic + " -> " + childs.ToMultiString(", "));
            SortedSet<Partition> mergedBrokerParts = updatedBrokerParts;
            if (this.actualBrokerTopicsPartitionsMap.ContainsKey(topic))
            {
                SortedSet<Partition> oldBrokerParts = this.actualBrokerTopicsPartitionsMap[topic];
                Logger.Debug(
                    "Unregistered list of brokers for topic " + topic + " -> " + oldBrokerParts.ToMultiString(", "));
                foreach (var oldBrokerPart in oldBrokerParts)
                {
                    mergedBrokerParts.Add(oldBrokerPart);
                }
            }
            else
            {
                this.actualBrokerTopicsPartitionsMap.Add(topic, null);
            }

            this.actualBrokerTopicsPartitionsMap[topic] = new SortedSet<Partition>(mergedBrokerParts.Where(x => this.actualBrokerIdMap.ContainsKey(x.BrokerId)));
        }
        /// <summary>
        /// Force get topic metadata and update 
        /// </summary>
        public void UpdateInfo(short versionId, int correlationId, string clientId, string topic)
        {
            Logger.InfoFormat("Will update metadata for topic:{0}", topic);
            Guard.NotNullNorEmpty(topic, "topic");
            var shuffledBrokers = this.syncProducerPool.GetShuffledProducers();
            var i = 0;
            var hasFetchedInfo = false;
            while (i < shuffledBrokers.Count && !hasFetchedInfo)
            {
                ISyncProducer producer = shuffledBrokers[i++];

                try
                {
                    var topicMetadataRequest = TopicMetadataRequest.Create(new List<string>() { topic }, versionId,
                                                                           correlationId, clientId);
                    var topicMetadataList = producer.Send(topicMetadataRequest);
                    var topicMetadata = topicMetadataList.Any() ? topicMetadataList.First() : null;
                    if (topicMetadata != null)
                    {
                        if (topicMetadata.Error != ErrorMapping.NoError)
                        {
                            Logger.WarnFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, topicMetadata.Error.ToString());
                        }
                        else
                        {
                            this.topicPartitionInfo[topic] = topicMetadata;
                            this.topicPartitionInfoLastUpdateTime[topic] = DateTime.UtcNow;
                            Logger.InfoFormat("Will  Update  metadata info, topic {0} ", topic);

                            //TODO:  For all partitions which has metadata, here return the sorted list.
                            //But sometimes kafka didn't return metadata for all topics.
                            this.topicPartitionInfoList[topic] = topicMetadata.PartitionsMetadata.Select(m =>
                                                        {
                                                            Partition partition = new Partition(topic, m.PartitionId);
                                                            if (m.Leader != null)
                                                            {
                                                                var leaderReplica = new Replica(m.Leader.Id, topic);
                                                                partition.Leader = leaderReplica;
                                                                Logger.InfoFormat("Topic {0} partition {1} has leader {2}", topic,
                                                                                   m.PartitionId, m.Leader.Id);

                                                                return partition;
                                                            }

                                                            Logger.WarnFormat("Topic {0} partition {1} does not have a leader yet", topic,
                                                                m.PartitionId);

                                                            return partition;
                                                        }
                                                        ).OrderBy(x => x.PartId).ToList(); ;
                            hasFetchedInfo = true;
                            Logger.InfoFormat("Finish  Update  metadata info, topic {0}  Partitions:{1}  No leader:{2}", topic, this.topicPartitionInfoList[topic].Count, this.topicPartitionInfoList[topic].Where(r => r.Leader == null).Count());

                            //In very weired case, the kafka broker didn't return metadata of all broker. need break and retry.  https://issues.apache.org/jira/browse/KAFKA-1998
                            // http://qnalist.com/questions/5899394/topicmetadata-response-miss-some-partitions-information-sometimes
                            if (zkClient != null)
                            {
                                Dictionary<int, int[]> topicMetaDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(this.zkClient, topic);
                                if (topicMetaDataInZookeeper != null && topicMetaDataInZookeeper.Any())
                                {
                                    topicDataInZookeeper[topic] = topicMetaDataInZookeeper;
                                    if (this.topicPartitionInfoList[topic].Count != topicMetaDataInZookeeper.Count)
                                    {
                                        Logger.ErrorFormat("NOT all partition has metadata.  Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count);
                                        throw new UnavailableProducerException(string.Format("Please make sure every partition at least has one broker running and retry again.   NOT all partition has metadata.  Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count));
                                    }
                                }
                            }

                        }
                    }
                }
                catch (Exception e)
                {
                    Logger.ErrorFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, e.FormatException());
                }
            }
        }