public void ZkAwareProducerSends1Message()
{
var prodConfig = this.ZooKeeperBasedSyncProdConfig;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData"));
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 });
var mockPartitioner = new MockAlwaysZeroPartitioner();
using (var producer = new Producer<string, Message>(prodConfig, mockPartitioner, new DefaultEncoder()))
{
var producerData = new ProducerData<string, Message>(
CurrentTestTopic, "somekey", new List<Message> { originalMessage });
producer.Send(producerData);
while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }))
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfiguration(
multipleBrokersHelper.BrokerThatHasChanged.Host,
multipleBrokersHelper.BrokerThatHasChanged.Port);
IConsumer consumer = new Consumer(consumerConfig);
var request = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null & response.Messages.Count() > 0)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString());
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Consumer"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
public Consumer(ConsumerConfiguration config)
{
Guard.NotNull(config, "config");
this.config = config;
this.host = config.Broker.Host;
this.port = config.Broker.Port;
}
public static long GetCurrentKafkaOffset(string topic, string address, int port, int partition)
{
var request = new OffsetRequest(topic, partition, DateTime.Now.AddDays(-5).Ticks, 10);
var consumerConfig = new ConsumerConfiguration(address, port);
IConsumer consumer = new Consumer(consumerConfig, address, port);
IList<long> list = consumer.GetOffsetsBefore(request);
return list.Sum();
}
/// <summary>
/// Initializes a new instance of the <see cref="Consumer"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
/// <param name="host"></param>
/// <param name="port"></param>
public Consumer(ConsumerConfiguration config, string host, int port)
{
Guard.NotNull(config, "config");
this.config = config;
this.host = host;
this.port = port;
}
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos)
{
this.name = name;
this.zkClient = zkClient;
this.config = config;
this.broker = broker;
this.partitionTopicInfos = partitionTopicInfos;
this.simpleConsumer = new Consumer(this.config, broker.Host, broker.Port);
}
/// <summary>
/// Initializes a new instance of the <see cref="ZookeeperConsumerConnector"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration. At the minimum, need to specify the group ID
/// of the consumer and the ZooKeeper connection string.
/// </param>
/// <param name="enableFetcher">
/// Indicates whether fetchers should be enabled
/// </param>
public ZookeeperConsumerConnector(ConsumerConfiguration config, bool enableFetcher)
{
this.config = config;
this.enableFetcher = enableFetcher;
this.ConnectZk();
this.CreateFetcher();
if (this.config.AutoCommit)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "starting auto committer every {0} ms", this.config.AutoCommitInterval);
scheduler.ScheduleWithRate(this.AutoCommit, this.config.AutoCommitInterval, this.config.AutoCommitInterval);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Consumer"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
public Consumer(ConsumerConfiguration config)
{
Guard.NotNull(config, "config");
this.config = config;
this.host = config.Broker.Host;
this.port = config.Broker.Port;
this.connection = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SendTimeout,
this.config.ReceiveTimeout,
this.config.ReconnectInterval);
this.CreatedTimeInUTC = DateTime.UtcNow.Ticks;
}
internal ZKRebalancerListener(
ConsumerConfiguration config,
string consumerIdString,
IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry,
IZooKeeperClient zkClient,
ZookeeperConsumerConnector zkConsumerConnector,
IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues,
Fetcher fetcher,
object syncLock)
{
this.syncLock = syncLock;
this.consumerIdString = consumerIdString;
this.config = config;
this.topicRegistry = topicRegistry;
this.zkClient = zkClient;
this.dirs = new ZKGroupDirs(config.GroupId);
this.zkConsumerConnector = zkConsumerConnector;
this.queues = queues;
this.fetcher = fetcher;
}
public ConsumerConfiguration(ConsumerConfiguration cosumerConfigTemplate, BrokerConfiguration brokerConfiguration)
{
this.Broker = brokerConfiguration;
this.NumberOfTries = cosumerConfigTemplate.NumberOfTries;
this.Timeout = cosumerConfigTemplate.Timeout;
this.AutoOffsetReset = cosumerConfigTemplate.AutoOffsetReset;
this.AutoCommit = cosumerConfigTemplate.AutoCommit;
this.AutoCommitInterval = cosumerConfigTemplate.AutoCommitInterval;
this.FetchSize = cosumerConfigTemplate.FetchSize;
this.MaxFetchFactor = cosumerConfigTemplate.MaxFetchFactor;
this.BackOffIncrement = cosumerConfigTemplate.BackOffIncrement;
this.ConsumerId = cosumerConfigTemplate.ConsumerId;
this.ReconnectInterval = cosumerConfigTemplate.ReconnectInterval;
this.ShutdownTimeout = cosumerConfigTemplate.ShutdownTimeout;
this.MaxFetchBufferLength = cosumerConfigTemplate.MaxFetchBufferLength;
this.SendTimeout = cosumerConfigTemplate.SendTimeout;
this.ReceiveTimeout = cosumerConfigTemplate.ReceiveTimeout;
this.BufferSize = cosumerConfigTemplate.BufferSize;
this.Verbose = cosumerConfigTemplate.Verbose;
this.ConsumeGroupRebalanceRetryIntervalMs = cosumerConfigTemplate.ConsumeGroupRebalanceRetryIntervalMs;
this.ConsumeGroupFindNewLeaderSleepIntervalMs = cosumerConfigTemplate.ConsumeGroupFindNewLeaderSleepIntervalMs;
}
public ConsumerConfiguration(ConsumerConfiguration cosumerConfigTemplate, BrokerConfiguration brokerConfiguration)
{
this.Broker = brokerConfiguration;
this.NumberOfTries = cosumerConfigTemplate.NumberOfTries;
this.Timeout = cosumerConfigTemplate.Timeout;
this.AutoOffsetReset = cosumerConfigTemplate.AutoOffsetReset;
this.AutoCommit = cosumerConfigTemplate.AutoCommit;
this.AutoCommitInterval = cosumerConfigTemplate.AutoCommitInterval;
this.FetchSize = cosumerConfigTemplate.FetchSize;
this.MaxFetchFactor = cosumerConfigTemplate.MaxFetchFactor;
this.BackOffIncrement = cosumerConfigTemplate.BackOffIncrement;
this.ConsumerId = cosumerConfigTemplate.ConsumerId;
this.ReconnectInterval = cosumerConfigTemplate.ReconnectInterval;
this.ShutdownTimeout = cosumerConfigTemplate.ShutdownTimeout;
this.MaxFetchBufferLength = cosumerConfigTemplate.MaxFetchBufferLength;
this.SendTimeout = cosumerConfigTemplate.SendTimeout;
this.ReceiveTimeout = cosumerConfigTemplate.ReceiveTimeout;
this.BufferSize = cosumerConfigTemplate.BufferSize;
this.Verbose = cosumerConfigTemplate.Verbose;
this.ConsumeGroupRebalanceRetryIntervalMs = cosumerConfigTemplate.ConsumeGroupRebalanceRetryIntervalMs;
this.ConsumeGroupFindNewLeaderSleepIntervalMs = cosumerConfigTemplate.ConsumeGroupFindNewLeaderSleepIntervalMs;
}
/// <summary>
/// Initializes a new instance of the <see cref="Fetcher"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
/// <param name="zkClient">
/// The wrapper above ZooKeeper client.
/// </param>
public Fetcher(ConsumerConfiguration config, IZooKeeperClient zkClient)
{
this.config = config;
this.zkClient = zkClient;
}
internal ConsumerGroupHelperUnit(int threadID, ConsumeGroupHelperOptions cg, AutoResetEvent e, int c)
{
this.resetEvent = e;
this.cgOptions = cg;
this.ThreadID = threadID;
this.Count = c;
configSettings = new ConsumerConfiguration
{
AutoOffsetReset = OffsetRequest.SmallestTime,
AutoCommit = false,
GroupId = cgOptions.ConsumerGroupName,
ConsumerId = cgOptions.ConsumerId + "_Thread_" + threadID.ToString(),
Timeout = cgOptions.Timeout,
ZooKeeper = new ZooKeeperConfiguration(cgOptions.Zookeeper, 30000, 4000, 8000),
BufferSize = cgOptions.BufferSize,
FetchSize = cgOptions.FetchSize,
MaxFetchBufferLength = cgOptions.MaxFetchBufferLength// cgOptions.FetchSize * (10~40) / cgOptions.MessageSize,
};
if (cgOptions.CancellationTimeoutMs != KafkaNETExampleConstants.DefaultCancellationTimeoutMs)
configSettings.Timeout = -1;
}
/// <summary>
/// clear the leader and configurations dependent on the leader
/// </summary>
public void ClearLeader()
{
if (this.producer != null)
{
this.producer.Dispose();
this.producer = null;
}
if (this.consumer != null)
{
this.consumer.Dispose();
this.consumer = null;
}
this.producerConf = null;
this.consumerConf = null;
this.leaders.Clear();
if (this.PartitionsMetadata != null)
{
this.PartitionsMetadata = null;
}
}
public void ProducerSends3Messages()
{
var prodConfig = this.ConfigBasedSyncProdConfig;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1"));
var originalMessage2 = new Message(Encoding.UTF8.GetBytes("TestData2"));
var originalMessage3 = new Message(Encoding.UTF8.GetBytes("TestData3"));
var originalMessageList = new List<Message> { originalMessage1, originalMessage2, originalMessage3 };
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets(
new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 });
using (var producer = new Producer(prodConfig))
{
var producerData = new ProducerData<string, Message>(CurrentTestTopic, originalMessageList);
producer.Send(producerData);
}
Thread.Sleep(waitSingle);
while (
!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(
new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }))
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfiguration(
multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port);
IConsumer consumer = new Consumer(consumerConfig);
var request = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 2)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(3, response.Messages.Count());
Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString());
Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString());
Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString());
}
public static long GetCurrentKafkaOffset(string topic, ConsumerConfiguration clientConfig)
{
return GetCurrentKafkaOffset(topic, clientConfig.Broker.Host, clientConfig.Broker.Port);
}
/// <summary>
/// Initializes a new instance of the <see cref="Consumer"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
/// <param name="host"></param>
/// <param name="port"></param>
public Consumer(ConsumerConfiguration config, string host, int port)
{
Guard.NotNull(config, "config");
this.config = config;
this.host = host;
this.port = port;
this.connection = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SendTimeout,
this.config.ReceiveTimeout,
this.config.ReconnectInterval);
}
public ClusterHealthChecker(ConsumerConfiguration config)
{
this.config = config;
this.ConnectZk();
}
public ConsumerOffsetChecker(ConsumerConfiguration config)
{
this.config = config;
this.ConnectZk();
}
public static PullResponse PullMessage(
ConsumerConfiguration consumerConfig,
BrokerConfiguration leaderBrokerConfig,
int correlationID,
string topic,
int partitionIndex,
long fetchOffset,
KafkaClientHelperConfiguration helperConfiguration,
out long offsetNew)
{
offsetNew = -1;
PullResponse result = null;
int payloadCount = 0;
// at least retry once
int maxRetry = 1;
int retryCount = 0;
string s = string.Empty;
bool success = false;
while (!success && retryCount < maxRetry)
{
try
{
var requestMap = new Dictionary<string, List<PartitionFetchInfo>>();
requestMap.Add(
topic,
new List<PartitionFetchInfo>()
{
new PartitionFetchInfo(
partitionIndex,
fetchOffset,
helperConfiguration.FetchSize)
});
using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port))
{
var response = consumer.Fetch(new FetchRequest(
correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id
helperConfiguration.MaxWaitTime,
helperConfiguration.MinWaitBytes,
requestMap));
if (response == null)
{
throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
var partitionData = response.PartitionData(topic, partitionIndex);
if (partitionData == null)
{
throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
{
s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return null;
}
if (partitionData.Error != ErrorMapping.NoError)
{
s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return null;
}
var messages = partitionData.MessageSet.Messages;
s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
success = true;
result = new PullResponse(partitionData);
if (null != messages && messages.Count() > 0)
{
payloadCount = messages.Count();
long lastOffset = messages.Last().Offset;
if ((payloadCount + fetchOffset) != (lastOffset + 1))
{
s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry);
}
offsetNew = messages.Last().Offset + 1;
}
return result;
}
}
catch (Exception)
{
if (retryCount >= maxRetry)
{
throw;
}
}
finally
{
retryCount++;
}
} // end of while loop
return result;
}
