示例#1
0
        public ConsumerFetcherManager(string consumerIdString, ConsumerConfig config, ZkClient zkClient)
            : base(
                string.Format("ConsumerFetcherManager-{0}", DateTimeHelper.CurrentTimeMilis()), config.ClientId, config.NumConsumerFetchers)
        {
            this.NoLeaderPartitionSet = new HashSet<TopicAndPartition>();
            this.consumerIdString = consumerIdString;
            this.config = config;
            this.zkClient = zkClient;

            this.@lock = new ReentrantLock();
            this.cond = [email protected]();
        }
示例#2
0
        public ConsumerFetcherThread(
            string name,
            ConsumerConfig config,
            Broker sourceBroker,
            IDictionary<TopicAndPartition, PartitionTopicInfo> partitionMap,
            ConsumerFetcherManager consumerFetcherManager) : base(
            name, 
            config.ClientId + "-" + name, 
            sourceBroker, 
            config.SocketTimeoutMs,
            config.SocketReceiveBufferBytes, 
            config.FetchMessageMaxBytes, 
            Request.OrdinaryConsumerId,
            config.FetchWaitMaxMs,
            config.FetchMinBytes, 
            true)

        {
            this.partitionMap = partitionMap;
            this.config = config;
            this.consumerFetcherManager = consumerFetcherManager;
        }
        public ConsumerPerfConfig(string[] args)
        {
            this.Args = args;

            this.GroupId = "perf-consumer" + new Random().Next(100000);
            this.FetchMessageMaxBytes = 1024 * 1024;
            this.AutoOffsetReset = false;
            this.SocketReceiveBufferBytes = 2 * 1024 * 1024;
            this.NumThreads = 10;
            this.NumConsumerFetchers = 1;

            OptionSet.Add(
                "zookeeper=",
                "REQUIRED: The connection string for the zookeeper connection in the form host:port. Multiple URLS can be given to allow fail-over.",
                b => this.ZookeeperConnect = b);

            OptionSet.Add("topic=", "REQUIRED: The topic to consume from.", t => this.Topic = t);

            OptionSet.Add("group:", "The group id to consume on.", g => this.GroupId = g);

            OptionSet.Add(
                "fetch-size:", "The amount of data to fetch in a single request.", f => this.FetchMessageMaxBytes = int.Parse(f));

            OptionSet.Add(
                "from-latest",
                "If the consumer does not already have an established offset to consume from, start with the latest message present in the log rather than the earliest message.",
                _ => this.AutoOffsetReset = true);

            OptionSet.Add(
                "socket-buffer-size:",
                "The size of the tcp RECV size.",
                s => this.SocketReceiveBufferBytes = int.Parse(s));

            OptionSet.Add("threads:", "Number of processing threads.", t => this.NumThreads = int.Parse(t));

            OptionSet.Add(
                "num-fetch-threads:", "Number of fetcher threads.", t => this.NumConsumerFetchers = int.Parse(t));

            this.ParseArguments();
            this.EnsureMinimalParameters();

            this.ConsumerConfig = new ConsumerConfig
                                      {
                                          GroupId = this.GroupId,
                                          SocketReceiveBufferBytes = this.SocketReceiveBufferBytes,
                                          FetchMessageMaxBytes = this.FetchMessageMaxBytes,
                                          AutoOffsetReset = this.AutoOffsetReset ? "largest" : "smallest",
                                          ZooKeeper = new ZkConfig { ZkConnect = this.ZookeeperConnect },
                                          ConsumerTimeoutMs = 5000,
                                          NumConsumerFetchers = this.NumConsumerFetchers
                                      };
        }
示例#4
0
 /// <summary>
 /// Create a ConsumerConnector
 /// </summary>
 /// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param>
 /// <returns></returns>
 public static IConsumerConnector Create(ConsumerConfig config)
 {
     var consumeConnect = new ZookeeperConsumerConnector(config);
     return consumeConnect;
 }
示例#5
0
        /// <summary>
        /// Create a ConsumerConnector
        /// </summary>
        /// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param>
        /// <returns></returns>
        public static IConsumerConnector Create(ConsumerConfig config)
        {
            var consumeConnect = new ZookeeperConsumerConnector(config);

            return(consumeConnect);
        }