public WildcardTopicCount(ZkClient zkClient, string consumerIdString, TopicFilter topicFilter, int numStreams) { this.ZkClient = zkClient; this.ConsumerIdString = consumerIdString; this.TopicFilter = topicFilter; this.NumStreams = numStreams; }
public IList <KafkaStream <TKey, TValue> > CreateMessageStreamsByFilter <TKey, TValue>( TopicFilter topicFilter, int numStreams = 1, IDecoder <TKey> keyDecoder = null, IDecoder <TValue> valueDecoder = null) { var wildcardStreamsHandler = new WildcardStreamsHandler <TKey, TValue>( this, topicFilter, numStreams, keyDecoder, valueDecoder); return(wildcardStreamsHandler.Streams()); }
public override IDictionary <string, ISet <string> > GetConsumerThreadIdsPerTopic() { var wildcardTopics = ZkUtils.GetChildrenParentMayNotExist(this.ZkClient, ZkUtils.BrokerTopicsPath) .Where(x => TopicFilter.IsTopicAllowed(x)) .ToList(); return(this.MakeConsumerThreadIdsPerTopic(this.ConsumerIdString, wildcardTopics.ToDictionary(x => x, v => this.NumStreams))); }
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient) { var dirs = new ZKGroupDirs(group); var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1; string subscriptionPattern; IDictionary <string, int> topMap; try { var parsedJson = JObject.Parse(topicCountString); if (parsedJson != null) { var pattern = parsedJson.Get("pattern"); if (pattern != null) { subscriptionPattern = pattern.Value <string>(); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } var topMapObject = (IEnumerable <KeyValuePair <string, JToken> >)parsedJson.Get("subscription"); if (topMapObject != null) { topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value <int>()); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } catch (Exception e) { Logger.Error("error parsing consumer json string " + topicCountString, e); throw; } var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern); var hasBlackList = BlackListPattern.Equals(subscriptionPattern); if (topMap.Count == 0 || !(hasWhiteList || hasBlackList)) { return(new StaticTopicCount(consumerId, topMap)); } else { var regex = topMap.First().Key; var numStreams = topMap.First().Value; TopicFilter filter = hasWhiteList ? (TopicFilter) new Whitelist(regex) : new Blacklist(regex); return(new WildcardTopicCount(zkClient, consumerId, filter, numStreams)); } }
internal WildcardStreamsHandler( ZookeeperConsumerConnector parent, TopicFilter topicFilter, int numStreams, IDecoder <TKey> keyDecoder, IDecoder <TValue> valueDecoder) { this.parent = parent; this.topicFilter = topicFilter; this.numStreams = numStreams; this.keyDecoder = keyDecoder; this.valueDecoder = valueDecoder; if (parent.messageStreamCreated.GetAndSet(true)) { throw new Exception("Each consumer connector can create message streams by filter at most once."); } this.wildcardQueuesAndStreams = Enumerable.Range(1, numStreams).Select(e => { var queue = new BlockingCollection <FetchedDataChunk>(this.parent.Config.QueuedMaxMessages); var stream = new KafkaStream <TKey, TValue>( queue, this.parent.Config.ConsumerTimeoutMs, keyDecoder, valueDecoder, this.parent.Config.ClientId); return(Tuple.Create(queue, stream)); }).ToList(); this.wildcardTopics = ZkUtils.GetChildrenParentMayNotExist(this.parent.zkClient, ZkUtils.BrokerTopicsPath) .Where(topicFilter.IsTopicAllowed) .ToList(); this.wildcardTopicCount = TopicCount.ConstructTopicCount( this.parent.consumerIdString, topicFilter, numStreams, this.parent.zkClient); this.dirs = new ZKGroupDirs(this.parent.Config.GroupId); this.parent.RegisterConsumerInZK(dirs, this.parent.consumerIdString, this.wildcardTopicCount); this.parent.ReinitializeConsumer(this.wildcardTopicCount, this.wildcardQueuesAndStreams); // Topic events will trigger subsequent synced rebalances. Logger.InfoFormat("Creating topic event watcher for topics {0}", topicFilter); this.parent.wildcardTopicWatcher = new ZookeeperTopicEventWatcher(this.parent.zkClient, this); }
public static WildcardTopicCount ConstructTopicCount( string consumerIdString, TopicFilter filter, int numStream, ZkClient zkClient) { return new WildcardTopicCount(zkClient, consumerIdString, filter, numStream); }
public static WildcardTopicCount ConstructTopicCount( string consumerIdString, TopicFilter filter, int numStream, ZkClient zkClient) { return(new WildcardTopicCount(zkClient, consumerIdString, filter, numStream)); }