public WildcardTopicCount(ZkClient zkClient, string consumerIdString, TopicFilter topicFilter, int numStreams)
{
this.ZkClient = zkClient;
this.ConsumerIdString = consumerIdString;
this.TopicFilter = topicFilter;
this.NumStreams = numStreams;
}
public IList <KafkaStream <TKey, TValue> > CreateMessageStreamsByFilter <TKey, TValue>(
TopicFilter topicFilter, int numStreams = 1, IDecoder <TKey> keyDecoder = null, IDecoder <TValue> valueDecoder = null)
{
var wildcardStreamsHandler = new WildcardStreamsHandler <TKey, TValue>(
this, topicFilter, numStreams, keyDecoder, valueDecoder);
return(wildcardStreamsHandler.Streams());
}
public override IDictionary <string, ISet <string> > GetConsumerThreadIdsPerTopic()
{
var wildcardTopics = ZkUtils.GetChildrenParentMayNotExist(this.ZkClient, ZkUtils.BrokerTopicsPath)
.Where(x => TopicFilter.IsTopicAllowed(x))
.ToList();
return(this.MakeConsumerThreadIdsPerTopic(this.ConsumerIdString, wildcardTopics.ToDictionary(x => x, v => this.NumStreams)));
}
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient)
{
var dirs = new ZKGroupDirs(group);
var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1;
string subscriptionPattern;
IDictionary <string, int> topMap;
try
{
var parsedJson = JObject.Parse(topicCountString);
if (parsedJson != null)
{
var pattern = parsedJson.Get("pattern");
if (pattern != null)
{
subscriptionPattern = pattern.Value <string>();
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
var topMapObject = (IEnumerable <KeyValuePair <string, JToken> >)parsedJson.Get("subscription");
if (topMapObject != null)
{
topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value <int>());
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
catch (Exception e)
{
Logger.Error("error parsing consumer json string " + topicCountString, e);
throw;
}
var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern);
var hasBlackList = BlackListPattern.Equals(subscriptionPattern);
if (topMap.Count == 0 || !(hasWhiteList || hasBlackList))
{
return(new StaticTopicCount(consumerId, topMap));
}
else
{
var regex = topMap.First().Key;
var numStreams = topMap.First().Value;
TopicFilter filter = hasWhiteList ? (TopicFilter) new Whitelist(regex) : new Blacklist(regex);
return(new WildcardTopicCount(zkClient, consumerId, filter, numStreams));
}
}
internal WildcardStreamsHandler(
ZookeeperConsumerConnector parent,
TopicFilter topicFilter,
int numStreams,
IDecoder <TKey> keyDecoder,
IDecoder <TValue> valueDecoder)
{
this.parent = parent;
this.topicFilter = topicFilter;
this.numStreams = numStreams;
this.keyDecoder = keyDecoder;
this.valueDecoder = valueDecoder;
if (parent.messageStreamCreated.GetAndSet(true))
{
throw new Exception("Each consumer connector can create message streams by filter at most once.");
}
this.wildcardQueuesAndStreams = Enumerable.Range(1, numStreams).Select(e =>
{
var queue = new BlockingCollection <FetchedDataChunk>(this.parent.Config.QueuedMaxMessages);
var stream = new KafkaStream <TKey, TValue>(
queue,
this.parent.Config.ConsumerTimeoutMs,
keyDecoder,
valueDecoder,
this.parent.Config.ClientId);
return(Tuple.Create(queue, stream));
}).ToList();
this.wildcardTopics =
ZkUtils.GetChildrenParentMayNotExist(this.parent.zkClient, ZkUtils.BrokerTopicsPath)
.Where(topicFilter.IsTopicAllowed)
.ToList();
this.wildcardTopicCount = TopicCount.ConstructTopicCount(
this.parent.consumerIdString, topicFilter, numStreams, this.parent.zkClient);
this.dirs = new ZKGroupDirs(this.parent.Config.GroupId);
this.parent.RegisterConsumerInZK(dirs, this.parent.consumerIdString, this.wildcardTopicCount);
this.parent.ReinitializeConsumer(this.wildcardTopicCount, this.wildcardQueuesAndStreams);
// Topic events will trigger subsequent synced rebalances.
Logger.InfoFormat("Creating topic event watcher for topics {0}", topicFilter);
this.parent.wildcardTopicWatcher = new ZookeeperTopicEventWatcher(this.parent.zkClient, this);
}
public static WildcardTopicCount ConstructTopicCount(
string consumerIdString, TopicFilter filter, int numStream, ZkClient zkClient)
{
return new WildcardTopicCount(zkClient, consumerIdString, filter, numStream);
}
public WildcardTopicCount(ZkClient zkClient, string consumerIdString, TopicFilter topicFilter, int numStreams)
{
this.ZkClient = zkClient;
this.ConsumerIdString = consumerIdString;
this.TopicFilter = topicFilter;
this.NumStreams = numStreams;
}
public static WildcardTopicCount ConstructTopicCount(
string consumerIdString, TopicFilter filter, int numStream, ZkClient zkClient)
{
return(new WildcardTopicCount(zkClient, consumerIdString, filter, numStream));
}
