public ConsumerFetcherManager(string consumerIdString, ConsumerConfig config, ZkClient zkClient)
: base(
string.Format("ConsumerFetcherManager-{0}", DateTimeHelper.CurrentTimeMilis()), config.ClientId, config.NumConsumerFetchers)
{
this.NoLeaderPartitionSet = new HashSet<TopicAndPartition>();
this.consumerIdString = consumerIdString;
this.config = config;
this.zkClient = zkClient;
this.@lock = new ReentrantLock();
this.cond = [email protected]();
}
public ConsumerFetcherThread(
string name,
ConsumerConfig config,
Broker sourceBroker,
IDictionary<TopicAndPartition, PartitionTopicInfo> partitionMap,
ConsumerFetcherManager consumerFetcherManager) : base(
name,
config.ClientId + "-" + name,
sourceBroker,
config.SocketTimeoutMs,
config.SocketReceiveBufferBytes,
config.FetchMessageMaxBytes,
Request.OrdinaryConsumerId,
config.FetchWaitMaxMs,
config.FetchMinBytes,
true)
{
this.partitionMap = partitionMap;
this.config = config;
this.consumerFetcherManager = consumerFetcherManager;
}
public ConsumerPerfConfig(string[] args)
{
this.Args = args;
this.GroupId = "perf-consumer" + new Random().Next(100000);
this.FetchMessageMaxBytes = 1024 * 1024;
this.AutoOffsetReset = false;
this.SocketReceiveBufferBytes = 2 * 1024 * 1024;
this.NumThreads = 10;
this.NumConsumerFetchers = 1;
OptionSet.Add(
"zookeeper=",
"REQUIRED: The connection string for the zookeeper connection in the form host:port. Multiple URLS can be given to allow fail-over.",
b => this.ZookeeperConnect = b);
OptionSet.Add("topic=", "REQUIRED: The topic to consume from.", t => this.Topic = t);
OptionSet.Add("group:", "The group id to consume on.", g => this.GroupId = g);
OptionSet.Add(
"fetch-size:", "The amount of data to fetch in a single request.", f => this.FetchMessageMaxBytes = int.Parse(f));
OptionSet.Add(
"from-latest",
"If the consumer does not already have an established offset to consume from, start with the latest message present in the log rather than the earliest message.",
_ => this.AutoOffsetReset = true);
OptionSet.Add(
"socket-buffer-size:",
"The size of the tcp RECV size.",
s => this.SocketReceiveBufferBytes = int.Parse(s));
OptionSet.Add("threads:", "Number of processing threads.", t => this.NumThreads = int.Parse(t));
OptionSet.Add(
"num-fetch-threads:", "Number of fetcher threads.", t => this.NumConsumerFetchers = int.Parse(t));
this.ParseArguments();
this.EnsureMinimalParameters();
this.ConsumerConfig = new ConsumerConfig
{
GroupId = this.GroupId,
SocketReceiveBufferBytes = this.SocketReceiveBufferBytes,
FetchMessageMaxBytes = this.FetchMessageMaxBytes,
AutoOffsetReset = this.AutoOffsetReset ? "largest" : "smallest",
ZooKeeper = new ZkConfig { ZkConnect = this.ZookeeperConnect },
ConsumerTimeoutMs = 5000,
NumConsumerFetchers = this.NumConsumerFetchers
};
}
/// <summary>
/// Create a ConsumerConnector
/// </summary>
/// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param>
/// <returns></returns>
public static IConsumerConnector Create(ConsumerConfig config)
{
var consumeConnect = new ZookeeperConsumerConnector(config);
return consumeConnect;
}
/// <summary>
/// Create a ConsumerConnector
/// </summary>
/// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param>
/// <returns></returns>
public static IConsumerConnector Create(ConsumerConfig config)
{
var consumeConnect = new ZookeeperConsumerConnector(config);
return(consumeConnect);
}
