public ZKSessionExpireListener(ZookeeperConsumerConnector parent, ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, IZKRebalancerListener loadbalancerListener)
{
this.parent = parent;
this.Dirs = dirs;
this.ConsumerIdString = consumerIdString;
this.TopicCount = topicCount;
this.LoadbalancerListener = loadbalancerListener;
}
public ZKSessionExpireListener(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, ZKRebalancerListener loadBalancerListener, ZookeeperConsumerConnector zkConsumerConnector)
{
this.consumerIdString = consumerIdString;
this.loadBalancerListener = loadBalancerListener;
this.zkConsumerConnector = zkConsumerConnector;
this.dirs = dirs;
this.topicCount = topicCount;
}
public ZKRebalancerListener(
ZookeeperConsumerConnector parent,
string group,
string consumerIdString,
IDictionary <string, IList <KafkaStream <TKey, TValue> > > kafkaMessageAndMetadataStreams)
{
this.parent = parent;
this.group = group;
this.consumerIdString = consumerIdString;
this.KafkaMessageAndMetadataStreams = kafkaMessageAndMetadataStreams;
this.@lock = new ReentrantLock();
this.cond = [email protected]();
this.watcherExecutorThread = new Thread(() =>
{
Logger.InfoFormat("starting watcher executor thread for consumer {0}", consumerIdString);
bool doRebalance;
while (!parent.isShuttingDown.Get())
{
try
{
[email protected]();
try
{
if (!isWatcherTriggered)
{
cond.Await(TimeSpan.FromMilliseconds(1000)); // wake up periodically so that it can check the shutdown flag
}
}
finally
{
doRebalance = isWatcherTriggered;
isWatcherTriggered = false;
@lock.Unlock();
}
if (doRebalance)
{
this.SyncedRebalance();
}
}
catch (Exception e)
{
Logger.Error("Error during syncedRebalance", e);
}
}
Logger.InfoFormat("Stoppping watcher executer thread for consumer {0}", consumerIdString);
});
this.watcherExecutorThread.Name = consumerIdString + "_watcher_executor";
this.watcherExecutorThread.Start();
}
public void ConsumerPerformsRebalancingWhenConsumerIsRemovedAndTakesItsPartitions()
{
var config = new ConsumerConfig(clientConfig)
{
AutoCommit = false,
GroupId = "group1",
ZkSessionTimeoutMs = 60000,
ZkConnectionTimeoutMs = 60000
};
IList<string> ids;
IList<string> owners;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
using (var consumerConnector2 = new ZookeeperConsumerConnector(config, true))
{
consumerConnector2.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
ids = client.GetChildren("/consumers/group1/ids", false).ToList();
owners = client.GetChildren("/consumers/group1/owners/test", false).ToList();
Assert.That(ids, Is.Not.Null.And.Not.Empty);
Assert.That(ids.Count, Is.EqualTo(2));
Assert.That(owners, Is.Not.Null.And.Not.Empty);
Assert.That(owners.Count, Is.EqualTo(2));
}
WaitUntillIdle(client, 1000);
ids = client.GetChildren("/consumers/group1/ids", false).ToList();
owners = client.GetChildren("/consumers/group1/owners/test", false).ToList();
Assert.That(ids, Is.Not.Null.And.Not.Empty);
Assert.That(ids.Count, Is.EqualTo(1));
Assert.That(owners, Is.Not.Null.And.Not.Empty);
Assert.That(owners.Count, Is.EqualTo(2));
var data1 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[0], false);
var data2 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[1], false);
Assert.That(data1, Is.Not.Null.And.Not.Empty);
Assert.That(data2, Is.Not.Null.And.Not.Empty);
Assert.That(data1, Is.EqualTo(data2));
Assert.That(data1, Is.StringStarting(ids[0]));
}
}
internal WildcardStreamsHandler(
ZookeeperConsumerConnector parent,
TopicFilter topicFilter,
int numStreams,
IDecoder <TKey> keyDecoder,
IDecoder <TValue> valueDecoder)
{
this.parent = parent;
this.topicFilter = topicFilter;
this.numStreams = numStreams;
this.keyDecoder = keyDecoder;
this.valueDecoder = valueDecoder;
if (parent.messageStreamCreated.GetAndSet(true))
{
throw new Exception("Each consumer connector can create message streams by filter at most once.");
}
this.wildcardQueuesAndStreams = Enumerable.Range(1, numStreams).Select(e =>
{
var queue = new BlockingCollection <FetchedDataChunk>(this.parent.Config.QueuedMaxMessages);
var stream = new KafkaStream <TKey, TValue>(
queue,
this.parent.Config.ConsumerTimeoutMs,
keyDecoder,
valueDecoder,
this.parent.Config.ClientId);
return(Tuple.Create(queue, stream));
}).ToList();
this.wildcardTopics =
ZkUtils.GetChildrenParentMayNotExist(this.parent.zkClient, ZkUtils.BrokerTopicsPath)
.Where(topicFilter.IsTopicAllowed)
.ToList();
this.wildcardTopicCount = TopicCount.ConstructTopicCount(
this.parent.consumerIdString, topicFilter, numStreams, this.parent.zkClient);
this.dirs = new ZKGroupDirs(this.parent.Config.GroupId);
this.parent.RegisterConsumerInZK(dirs, this.parent.consumerIdString, this.wildcardTopicCount);
this.parent.ReinitializeConsumer(this.wildcardTopicCount, this.wildcardQueuesAndStreams);
// Topic events will trigger subsequent synced rebalances.
Logger.InfoFormat("Creating topic event watcher for topics {0}", topicFilter);
this.parent.wildcardTopicWatcher = new ZookeeperTopicEventWatcher(this.parent.zkClient, this);
}
internal ZKRebalancerListener(
ConsumerConfiguration config,
string consumerIdString,
IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry,
IZooKeeperClient zkClient,
ZookeeperConsumerConnector zkConsumerConnector,
IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues,
Fetcher fetcher,
object syncLock)
{
this.syncLock = syncLock;
this.consumerIdString = consumerIdString;
this.config = config;
this.topicRegistry = topicRegistry;
this.zkClient = zkClient;
this.dirs = new ZKGroupDirs(config.GroupId);
this.zkConsumerConnector = zkConsumerConnector;
this.queues = queues;
this.fetcher = fetcher;
}
public void ConsumerPerformsRebalancingWhenNewConsumerIsAddedAndTheyDividePartitions()
{
var config = this.ZooKeeperBasedConsumerConfig;
IList<string> ids;
IList<string> owners;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>(
"zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
using (var consumerConnector2 = new ZookeeperConsumerConnector(config, true))
{
consumerConnector2.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
ids = client.GetChildren("/consumers/group1/ids", false).ToList();
owners = client.GetChildren("/consumers/group1/owners/test", false).ToList();
Assert.That(ids, Is.Not.Null.And.Not.Empty);
Assert.That(ids.Count, Is.EqualTo(2));
Assert.That(owners, Is.Not.Null.And.Not.Empty);
Assert.That(owners.Count, Is.EqualTo(2));
var data1 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[0], false);
var data2 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[1], false);
Assert.That(data1, Is.Not.Null.And.Not.Empty);
Assert.That(data2, Is.Not.Null.And.Not.Empty);
Assert.That(data1, Is.Not.EqualTo(data2));
Assert.That(data1, Is.StringStarting(ids[0]).Or.StringStarting(ids[1]));
Assert.That(data2, Is.StringStarting(ids[0]).Or.StringStarting(ids[1]));
}
}
}
public void TestBasic()
{
// test consumer timeout logic
var consumerConfig0 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer0, 200);
var zkConsumerConnector0 = new ZookeeperConsumerConnector(consumerConfig0);
var topicMessageSterams0 =
zkConsumerConnector0.CreateMessageStreams(
new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder());
// no messages to consume, we should hit timeout;
// also the iterator should support re-entrant, so loop it twice
for (var i = 0; i < 2; i++)
{
Assert.Throws<ConsumerTimeoutException>(
() => this.GetMessages(nMessages * 2, topicMessageSterams0));
}
zkConsumerConnector0.Shutdown();
// send some messages to each broker
var sentMessages1 =
this.SendMessagesToBrokerPartition(Configs.First(), Topic, 0, nMessages)
.Union(this.SendMessagesToBrokerPartition(Configs.Last(), Topic, 0, nMessages)).ToList();
// wait to make sure the topic and partition have a leader for the successful case
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 1, 1000);
// create a consuemr
var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1);
var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1);
var topicMessageStreams1 =
zkConsumerConnector1.CreateMessageStreams(
new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder());
var receivedMessages1 = this.GetMessages(nMessages * 2, topicMessageStreams1);
Assert.Equal(sentMessages1.OrderBy(x => x).ToArray(), receivedMessages1.OrderBy(x => x).ToArray());
// also check partition ownership
var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir);
var expected_1 = new List<Tuple<string, string>>
{
Tuple.Create("0", "group1_consumer1-0"),
Tuple.Create("1", "group1_consumer1-0")
};
Assert.Equal(expected_1, actual_1);
// commit consumer offsets
zkConsumerConnector1.CommitOffsets();
// create a consumer
var consumerConfig2 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer2);
consumerConfig2.RebalanceBackoffMs = RebalanceBackoutMs;
var zkConsumerConnector2 = new ZookeeperConsumerConnector(consumerConfig2);
var topicMessageStreams2 =
zkConsumerConnector2.CreateMessageStreams(
new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder());
// send some messages to each broker
var sentMessages2 =
this.SendMessagesToBrokerPartition(Configs.First(), Topic, 0, nMessages)
.Union(this.SendMessagesToBrokerPartition(Configs.Last(), Topic, 1, nMessages)).ToList();
// wait to make sure the topic and partition have a leader for the successful case
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500);
var receivedMessages2 =
this.GetMessages(nMessages, topicMessageStreams1)
.Union(this.GetMessages(nMessages, topicMessageStreams2))
.ToList();
Assert.Equal(sentMessages2.OrderBy(x => x).ToList(), receivedMessages2.OrderBy(x => x).ToList());
// also check partition ownership
var actual_2 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir);
var expected_2 = new List<Tuple<string, string>>
{
Tuple.Create("0", "group1_consumer1-0"),
Tuple.Create("1", "group1_consumer2-0")
};
Assert.Equal(expected_2, actual_2);
// create a consumer with empty map
var consumerConfig3 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer3);
var zkConsumerConnector3 = new ZookeeperConsumerConnector(consumerConfig3);
zkConsumerConnector3.CreateMessageStreams(new Dictionary<string, int>());
// send some messages to each broker
var sentMessages3 =
this.SendMessagesToBrokerPartition(Configs.First(), Topic, 0, nMessages)
.Union(this.SendMessagesToBrokerPartition(Configs.Last(), Topic, 1, nMessages))
.ToList();
// wait to make sure the topic and partition have a leader for the successful case
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500);
var receivedMessages3 =
this.GetMessages(nMessages, topicMessageStreams1)
.Union(this.GetMessages(nMessages, topicMessageStreams2))
.ToList();
Assert.Equal(sentMessages3.OrderBy(x => x).ToList(), receivedMessages3.OrderBy(x => x).ToList());
// also check partition ownership
var actual_3 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir);
Assert.Equal(expected_2, actual_3);
zkConsumerConnector1.Shutdown();
zkConsumerConnector2.Shutdown();
zkConsumerConnector3.Shutdown();
Logger.Info("all consumer connectors stopped");
}
/// <summary>
/// Create a ConsumerConnector
/// </summary>
/// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param>
/// <returns></returns>
public static IConsumerConnector Create(ConsumerConfig config)
{
var consumeConnect = new ZookeeperConsumerConnector(config);
return consumeConnect;
}
public void TestConsumerDecoder()
{
// send some messages to each broker
var sentMessages = this.SendMessagesToBrokerPartition(
Configs.First(), Topic, 0, nMessages)
.Union(
this.SendMessagesToBrokerPartition(
Configs.First(), Topic, 1, nMessages))
.ToList();
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 1, 1000);
// create a consuemr
var consumerConfig = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500);
var zkConsumerConnector = new ZookeeperConsumerConnector(consumerConfig);
var topicMessageStreams =
zkConsumerConnector.CreateMessageStreams(
new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder());
var receivedMessages = this.GetMessages(nMessages * 2, topicMessageStreams);
Assert.Equal(sentMessages.OrderBy(x => x).ToArray(), receivedMessages.OrderBy(x => x).ToArray());
zkConsumerConnector.Shutdown();
}
public void TestLeaderSelectionForPartition()
{
var zkClient = new ZkClient(this.zookeeperConnect, 6000, 30000, new ZkStringSerializer());
// create topic topic1 with 1 partition on broker 0
AdminUtils.CreateTopic(zkClient, Topic, 1, 1, new Dictionary<string, string>());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 3000);
var sentMessages1 = this.SendMessages(
Configs.First(), nMessages, "batch1", CompressionCodecs.NoCompressionCodec, 1);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000);
// create a consuemr
var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1);
var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1);
var topicMessageStreams1 =
zkConsumerConnector1.CreateMessageStreams(
new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder());
var topicRegistry = zkConsumerConnector1.TopicRegistry;
Assert.Equal(1, topicRegistry.Select(x => x.Key).Count());
Assert.Equal(Topic, topicRegistry.Select(x => x.Key).First());
var topicsAndPartitionsInRegistry =
topicRegistry.Select(x => Tuple.Create(x.Key, x.Value.Select(p => p.Value))).ToList();
var brokerPartition = topicsAndPartitionsInRegistry.First().Item2.First();
Assert.Equal(0, brokerPartition.PartitionId);
// also check partition ownership
var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir);
var expected_1 = new List<Tuple<string, string>>
{
Tuple.Create("0", "group1_consumer1-0"),
};
Assert.Equal(expected_1, actual_1);
var receivedMessages1 = this.GetMessages(nMessages, topicMessageStreams1);
Assert.Equal(sentMessages1, receivedMessages1);
zkConsumerConnector1.Shutdown();
zkClient.Dispose();
}
internal void Consume()
{
// connects to zookeeper
using (ZookeeperConsumerConnector connector = new ZookeeperConsumerConnector(configSettings, true))
{
if (this.ThreadID == 0)
{
ConsumerGroupHelper.initialOffset = connector.GetOffset(cgOptions.Topic);
Logger.InfoFormat("======Original offset \r\n{0}", ConsumerGroupHelper.initialOffset == null ? "(NULL)" : ConsumeGroupMonitorHelper.GetComsumerGroupOffsetsAsLog(ConsumerGroupHelper.initialOffset));
}
// defines collection of topics and number of threads to consume it with
// ===============NOTE============================
// For example , if there is 80 partitions for one topic.
//
// Normally start more than 96 = 80*120% clients with same GroupId. ( the extra 20% are buffer for autopilot IMP). And set FetchThreadCountPerConsumer as 1.
// Then 80 clients can lock partitions, can set MACHINENAME_ProcessID as ConsumerId, other 16 are idle. Strongly recomand take this method.
//
// If start 40 clients, and set FetchThreadCountPerConsumer as 1. then every client can lock 2 partitions at least. And if some client not available for autopilot
// IMP reason, then some of the client maybe lock 3 partitions.
//
// If start 40 clients, and set FetchThreadCountPerConsumer as 2, you will get two IEnumerator<Message>:topicData[0].GetEnumerator(),topicData[1].GetEnumerator()
// you need start TWO threads to process them in dependently.
// If the client get 2 partitions, each thread will handle 1 partition,
// If the client get 3 partitions, then one thread get 2 partitions, the other one get 1 partition. It will make the situaiton complex and the consume of partition not balance.
//==================NOTE=============================
IDictionary<string, int> topicMap = new Dictionary<string, int> { { cgOptions.Topic, cgOptions.FetchThreadCountPerConsumer } };
// get references to topic streams.
IDictionary<string, IList<KafkaMessageStream<Message>>> streams = connector.CreateMessageStreams(topicMap, new DefaultDecoder());
IList<KafkaMessageStream<Message>> topicData = streams[cgOptions.Topic];
long latestTotalCount = 0;
bool hitEndAndCommited = false;
if (cgOptions.CancellationTimeoutMs == KafkaNETExampleConstants.DefaultCancellationTimeoutMs)
{
// Get the message enumerator.
IEnumerator<Message> messageEnumerator = topicData[0].GetEnumerator();
//TODO: the enumerator count equal with FetchThreadCountPerConsumer . For example, if that value is 5, then here should get 5 enumerator.
//IF have 100 partitions, and only 20 consumers, need set this value to 5. and start 5 thread handle each one.
// Add tuples until maximum receive message count is reached or no new messages read after consumer configured timeout.
while (true)
{
bool noMoreMessage = false;
try
{
messageEnumerator.MoveNext();
Message m = messageEnumerator.Current;
latestTotalCount = Interlocked.Increment(ref ConsumerGroupHelper.totalCount);
Logger.InfoFormat("Message {0} from Partition:{1}, Offset:{2}, key:{3}, value:{4}", latestTotalCount, m.PartitionId, m.Offset, m.Key == null ? "(null)" : Encoding.UTF8.GetString(m.Key), m.Payload == null ? "(null)" : Encoding.UTF8.GetString(m.Payload));
if (latestTotalCount == 1)
{
Logger.InfoFormat("Read FIRST message, it's offset: {0} PartitionID:{1}", m.Offset, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.PartitionId);
}
hitEndAndCommited = false;
if (latestTotalCount % cgOptions.CommitBatchSize == 0)
{
//NOTE======
//Normally, just directly call .CommitOffsets()
// CommitOffset(string topic, int partition, long offset) only used when customer has strong requirement for reprocess messages as few as possible.
//Need tune the frequecy of calling .CommitOffsets(), it will directly increate zookeeper load and impact your overall performance
if (cgOptions.CommitOffsetWithPartitionIDOffset)
connector.CommitOffset(cgOptions.Topic, m.PartitionId.Value, m.Offset);
else
connector.CommitOffsets();
Console.WriteLine("\tRead some and commit once, LATEST message offset: {0}. PartitionID:{1} -- {2} Totally read {3} will commit offset. {4} FetchOffset:{5} ConsumeOffset:{6} CommitedOffset:{7}"
, m.Offset, m.PartitionId.Value, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.PartitionId, latestTotalCount, DateTime.Now
, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.FetchOffset
, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.ConsumeOffset
, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.CommitedOffset);
}
if (cgOptions.Count > 0 && latestTotalCount >= cgOptions.Count)
{
Logger.InfoFormat("Read LAST message, it's offset: {0}. PartitionID:{1} Totally read {2} want {3} will exit.", m.Offset, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.PartitionId, latestTotalCount, cgOptions.Count);
break;
}
}
catch (ConsumerTimeoutException)
{
if (!hitEndAndCommited)
{
Logger.InfoFormat("Totally Read {0} will commit offset. {1}", latestTotalCount, DateTime.Now);
connector.CommitOffsets();
hitEndAndCommited = true;
}
// Thrown if no new messages read after consumer configured timeout.
noMoreMessage = true;
}
if (noMoreMessage)
{
Logger.InfoFormat("No more message , hit end ,will Sleep(1), {0}", DateTime.Now);
if (cgOptions.SleepTypeWhileAlwaysRead == 0)
Thread.Sleep(0);
else if (cgOptions.SleepTypeWhileAlwaysRead == 1)
Thread.Sleep(1); //Best choice is Thread.Sleep(1). Other 3 choice still make the CPU 100%
else if (cgOptions.SleepTypeWhileAlwaysRead == 2)
Thread.Yield();
else
{
}
}
}
}
else
{
//Siphon scenario, repeatly take some messages and process. if no enough messages, will stop current batch after timeout.
while (true)
{
#if NET45
bool noMoreMessage = false;
Message lastMessage = null;
int count = 0;
KafkaMessageStream<Message> messagesStream = null;
ConsumerIterator<Message> iterator = null;
using (CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(cgOptions.CancellationTimeoutMs))
{
lastMessage = null;
IEnumerable<Message> messages = topicData[0].GetCancellable(cancellationTokenSource.Token);
messagesStream = (KafkaMessageStream<Message>)messages;
iterator = (ConsumerIterator<Message>)messagesStream.iterator;
foreach (Message message in messages)
{
latestTotalCount = Interlocked.Increment(ref ConsumerGroupHelper.totalCount);
lastMessage = message;
if (latestTotalCount == 1)
{
PartitionTopicInfo p = iterator.currentTopicInfo;
Logger.InfoFormat("Read FIRST message, it's offset: {0} PartitionID:{1}", lastMessage.Offset, p == null ? "null" : p.PartitionId.ToString());
}
hitEndAndCommited = false;
if (++count >= cgOptions.CommitBatchSize)
{
cancellationTokenSource.Cancel();
}
}
}
if (count > 0)
{
connector.CommitOffsets();
consumedTotalCount += count;
PartitionTopicInfo p = iterator.currentTopicInfo;
Console.WriteLine("\tRead some and commit once, Thread: {8} consumedTotalCount:{9} Target:{10} LATEST message offset: {0}. PartitionID:{1} -- {2} Totally read {3} will commit offset. {4} FetchOffset:{5} ConsumeOffset:{6} CommitedOffset:{7}"
, lastMessage.Offset, lastMessage.PartitionId.Value, p == null ? "null" : p.PartitionId.ToString(), latestTotalCount, DateTime.Now
, p == null ? "null" : p.FetchOffset.ToString()
, p == null ? "null" : p.ConsumeOffset.ToString()
, p == null ? "null" : p.CommitedOffset.ToString()
, this.ThreadID
, this.consumedTotalCount
, this.Count);
}
else
{
noMoreMessage = true;
}
if (this.Count > 0 && consumedTotalCount >= this.Count)
{
Logger.InfoFormat("Current thrad Read LAST message, Totally read {0} want {1} will exit current thread.", consumedTotalCount, this.Count);
break;
}
if (noMoreMessage)
{
Logger.InfoFormat("No more message , hit end ,will Sleep(2000), {0}", DateTime.Now);
if (cgOptions.SleepTypeWhileAlwaysRead == 0)
Thread.Sleep(0);
else if (cgOptions.SleepTypeWhileAlwaysRead == 1)
Thread.Sleep(2000); //Best choice is Thread.Sleep(1). Other 3 choice still make the CPU 100%
else if (cgOptions.SleepTypeWhileAlwaysRead == 2)
Thread.Yield();
else
{
}
}
#endif
#if NET4
throw new NotSupportedException("Please use .net45 to compile .");
#endif
}
}
Logger.InfoFormat("Read {0} will commit offset. {1}", latestTotalCount, DateTime.Now);
connector.CommitOffsets();
latestTotalCount = Interlocked.Read(ref ConsumerGroupHelper.totalCount);
Logger.InfoFormat("Totally read {0} want {1} . ", latestTotalCount, cgOptions.Count);
if (this.ThreadID == 0)
{
ConsumerGroupHelper.newOffset = connector.GetOffset(cgOptions.Topic);
}
}
this.resetEvent.Set();
}
public void TestCompressionSetConsumption()
{
// send some messages to each broker
var sentMessages = this.SendMessagesToBrokerPartition(
Configs.First(), Topic, 0, 200, CompressionCodecs.DefaultCompressionCodec)
.Union(
this.SendMessagesToBrokerPartition(
Configs.First(), Topic, 1, 200, CompressionCodecs.DefaultCompressionCodec))
.ToList();
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 1, 1000);
// create a consuemr
var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer0);
var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1);
var topicMessageStreams1 =
zkConsumerConnector1.CreateMessageStreams(
new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder());
var receivedMessages1 = this.GetMessages(400, topicMessageStreams1);
Assert.Equal(sentMessages.OrderBy(x => x).ToArray(), receivedMessages1.OrderBy(x => x).ToArray());
// also check partition ownership
var actual_2 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir);
var expected_2 = new List<Tuple<string, string>>
{
Tuple.Create("0", "group1_consumer0-0"),
Tuple.Create("1", "group1_consumer0-0")
};
Assert.Equal(expected_2, actual_2);
zkConsumerConnector1.Shutdown();
}
public ZKTopicPartitionChangeListener(
ZookeeperConsumerConnector parent, IZKRebalancerListener loadBalancerListener)
{
this.parent = parent;
this.LoadbalancerListener = loadBalancerListener;
}
public void ConsumerPorformsRebalancingWhenBrokerIsRemovedFromTopic()
{
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 };
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
client.CreateEphemeral(brokerTopicPath, 1);
WaitUntillIdle(client, 1000);
client.DeleteRecursive(brokerTopicPath);
WaitUntillIdle(client, 1000);
IList<string> children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children.Count, Is.EqualTo(2));
Assert.That(children, Has.None.EqualTo("2345-0"));
var topicRegistry = ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>("topicRegistry", consumerConnector);
Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty);
Assert.That(topicRegistry.Count, Is.EqualTo(1));
var item = topicRegistry["test"];
Assert.That(item.Count, Is.EqualTo(2));
Assert.That(item.Where(x => x.Value.BrokerId == 2345).Count(), Is.EqualTo(0));
}
}
public void ConsumerConnectorReceivesAShutdownSignal()
{
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
// putting the shutdown command into the queue
FieldInfo fi = typeof(ZookeeperConsumerConnector).GetField(
"queues", BindingFlags.NonPublic | BindingFlags.Instance);
var value =
(IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>>)
fi.GetValue(consumerConnector);
foreach (var topicConsumerQueueMap in value)
{
topicConsumerQueueMap.Value.Add(ZookeeperConsumerConnector.ShutdownCommand);
}
var sets = messages[CurrentTestTopic];
var resultMessages = new List<Message>();
foreach (var set in sets)
{
foreach (var message in set)
{
resultMessages.Add(message);
}
}
Assert.AreEqual(0, resultMessages.Count);
}
}
public void ConsumerPorformsRebalancingWhenNewBrokerIsAddedToTopic()
{
var config = new ConsumerConfig(clientConfig)
{
AutoCommit = false,
GroupId = "group1",
ZkSessionTimeoutMs = 60000,
ZkConnectionTimeoutMs = 60000
};
string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345;
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>(
"zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
IList<string> children = client.GetChildren("/consumers/group1/ids", false);
string consumerId = children[0];
client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
client.CreateEphemeral(brokerTopicPath, 1);
WaitUntillIdle(client, 500);
children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children.Count, Is.EqualTo(3));
Assert.That(children, Contains.Item("2345-0"));
string data = client.ReadData<string>("/consumers/group1/owners/test/2345-0");
Assert.That(data, Is.Not.Null);
Assert.That(data, Contains.Substring(consumerId));
var topicRegistry =
ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>(
"topicRegistry", consumerConnector);
Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty);
Assert.That(topicRegistry.Count, Is.EqualTo(1));
var item = topicRegistry["test"];
Assert.That(item.Count, Is.EqualTo(3));
var broker = topicRegistry["test"].SingleOrDefault(x => x.Key.BrokerId == 2345);
Assert.That(broker, Is.Not.Null);
}
}
public void ConsumerPorformsRebalancingOnStart()
{
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1" };
using (var consumerConnector = new ZookeeperConsumerConnector(config, true))
{
ZooKeeperClient client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector);
Assert.IsNotNull(client);
client.DeleteRecursive("/consumers/group1");
var topicCount = new Dictionary<string, int> { { "test", 1 } };
consumerConnector.CreateMessageStreams(topicCount);
WaitUntillIdle(client, 1000);
IList<string> children = client.GetChildren("/consumers", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("group1"));
children = client.GetChildren("/consumers/group1", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children, Contains.Item("ids"));
Assert.That(children, Contains.Item("owners"));
children = client.GetChildren("/consumers/group1/ids", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
string consumerId = children[0];
children = client.GetChildren("/consumers/group1/owners", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(1));
Assert.That(children, Contains.Item("test"));
children = client.GetChildren("/consumers/group1/owners/test", false);
Assert.That(children, Is.Not.Null.And.Not.Empty);
Assert.That(children.Count, Is.EqualTo(2));
string partId = children[0];
string data = client.ReadData<string>("/consumers/group1/owners/test/" + partId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Contains.Substring(consumerId));
data = client.ReadData<string>("/consumers/group1/ids/" + consumerId);
Assert.That(data, Is.Not.Null.And.Not.Empty);
Assert.That(data, Is.EqualTo("{ \"test\": 1 }"));
}
using (var client = new ZooKeeperClient(config.ZkConnect, config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
{
client.Connect();
//// Should be created as ephemeral
IList<string> children = client.GetChildren("/consumers/group1/ids");
Assert.That(children, Is.Null.Or.Empty);
//// Should be created as ephemeral
children = client.GetChildren("/consumers/group1/owners/test");
Assert.That(children, Is.Null.Or.Empty);
}
}
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 };
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
KafkaMessageStream myStream = sets[0];
var enumerator = myStream.GetEnumerator();
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString());
Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext());
Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state
enumerator.Reset();
// producing again
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
Thread.Sleep(3000);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString());
}
}
/// <summary>
/// Create a ConsumerConnector
/// </summary>
/// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param>
/// <returns></returns>
public static IConsumerConnector Create(ConsumerConfig config)
{
var consumeConnect = new ZookeeperConsumerConnector(config);
return(consumeConnect);
}
public void ConsumerConnectorConsumesTwoDifferentTopics()
{
string topic1 = CurrentTestTopic + "1";
string topic2 = CurrentTestTopic + "2";
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 });
producer.Send(producerRequest1);
var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages1 = new List<Message>();
var resultMessages2 = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
Assert.IsTrue(messages.ContainsKey(topic1));
Assert.IsTrue(messages.ContainsKey(topic2));
var sets1 = messages[topic1];
try
{
foreach (var set in sets1)
{
foreach (var message in set)
{
resultMessages1.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
var sets2 = messages[topic2];
try
{
foreach (var set in sets2)
{
foreach (var message in set)
{
resultMessages2.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(1, resultMessages1.Count);
Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString());
Assert.AreEqual(1, resultMessages2.Count);
Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString());
}
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
try
{
foreach (var set in sets)
{
foreach (var message in set)
{
resultMessages.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(2, resultMessages.Count);
Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString());
Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString());
}