public ZookeeperConsumerConnector(ConsumerConfig config, bool enableFetcher = true)
{
this.Config = config;
this.EnableFetcher = enableFetcher;
string consumerUuid;
if (config.ConsumerId != null)
{
consumerUuid = config.ConsumerId;
}
else
{
// generate unique consumerId automatically
var uuid = Guid.NewGuid();
consumerUuid = string.Format(
"{0}-{1}-{2}", Dns.GetHostName(), DateTimeHelper.CurrentTimeMilis(), BitConverter.ToString(uuid.ToByteArray()).Substring(0, 8));
}
this.consumerIdString = config.GroupId + "_" + consumerUuid;
this.logIdent = "[" + this.consumerIdString + "]";
this.ConnectZk();
this.CreateFetcher();
if (config.AutoCommitEnable)
{
this.scheduler.Startup();
Logger.InfoFormat("starting auto committer every {0} ms", config.AutoCommitIntervalMs);
this.scheduler.Schedule("kafka-consumer-autocommit", this.AutoCommit, TimeSpan.FromMilliseconds(config.AutoCommitIntervalMs), TimeSpan.FromMilliseconds(config.AutoCommitIntervalMs));
}
KafkaMetricsReporter.StartReporters(this.Config);
}
public Producer(ProducerConfig config, IEventHandler <TKey, TValue> eventHandler)
{
this.config = config;
this.eventHandler = eventHandler;
this.queue = new BlockingCollection <KeyedMessage <TKey, TValue> >(config.QueueBufferingMaxMessages);
if (config.ProducerType == ProducerTypes.Async)
{
this.sync = false;
this.producerSendThread = new ProducerSendThread <TKey, TValue>("ProducerSendThread-" + config.ClientId, this.queue, eventHandler, config.QueueBufferingMaxMs, config.BatchNumMessages, config.ClientId);
this.producerSendThread.Start();
}
this.producerTopicStats = ProducerTopicStatsRegistry.GetProducerTopicStats(config.ClientId);
KafkaMetricsReporter.StartReporters(this.config);
}
public ProducerPerfConfig(string[] args)
{
this.Args = args;
this.ProducerRequestTimeoutMs = 3000;
this.ProducerNumRetries = 3;
this.ProducerRetryBackoffMs = 100;
this.ProducerRequestRequiredAcks = -1;
this.NumThreads = 1;
this.MessageSendGapMs = 0;
this.IsFixedSize = true;
OptionSet.Add(
"broker-list=",
"REQUIRED: broker info (the list of broker host and port for bootstrap.",
b => this.BrokerList = b);
OptionSet.Add(
"topics=", "REQUIRED: The comma separated list of topics to produce to", t => this.TopicsStr = t);
OptionSet.Add(
"request-timeout-ms:",
"The produce request timeout in ms",
t => this.ProducerRequestTimeoutMs = int.Parse(t));
OptionSet.Add(
"producer-num-retries:", "The producer retries number", x => this.ProducerNumRetries = int.Parse(x));
OptionSet.Add(
"producer-retry-backoff-ms:",
"The producer retry backoff time in milliseconds",
x => this.ProducerRetryBackoffMs = int.Parse(x));
OptionSet.Add(
"request-num-acks:",
"Number of acks required for producer request to complete",
x => this.ProducerRequestRequiredAcks = short.Parse(x));
OptionSet.Add(
"vary-message-size",
"If set, message size will vary up to the given maximum.",
x => this.IsFixedSize = false);
OptionSet.Add("sync", "If set, messages are sent synchronously.", x => this.IsSync = true);
OptionSet.Add("threads:", "Number of sending threads.", x => this.NumThreads = int.Parse(x));
OptionSet.Add(
"initial-message-id:",
"The is used for generating test data, If set, messages will be tagged with an ID and sent by producer starting from this ID sequentially. Message content will be String type and in the form of 'Message:000...1:xxx...'",
x => this.InitialMessageId = int.Parse(x));
OptionSet.Add(
"message-send-gap-ms:",
"If set, the send thread will wait for specified time between two sends",
x => this.MessageSendGapMs = int.Parse(x));
OptionSet.Add(
"csv-reporter-enabled",
"If set, the CSV metrics reporter will be enabled",
x => this.CsvMetricsReporterEnabled = true);
OptionSet.Add(
"metrics-dir:",
"If csv-reporter-enable is set, and this parameter is" + "set, the csv metrics will be outputed here",
x => this.MetricsDirectory = x);
this.ParseArguments();
this.EnsureMinimalParameters();
if (this.CsvMetricsReporterEnabled)
{
var props = new Dictionary <string, string>();
props["kafka.metrics.polling.interval.secs"] = "1";
props["kafka.metrics.reporters"] = "kafka.metrics.KafkaCSVMetricsReporter";
if (string.IsNullOrEmpty(this.MetricsDirectory))
{
props["kafka.csv.metrics.dir"] = this.MetricsDirectory;
}
else
{
props["kafka.csv.metrics.dir"] = "kafka_metrics";
}
props["kafka.csv.metrics.reporter.enabled"] = "true";
KafkaMetricsReporter.StartReporters(new Config()); //TODO: change me!
}
}