private void AddPartitionTopicInfo( Pool <string, Pool <int, PartitionTopicInfo> > currentTopicRegistry, ZKGroupTopicDirs topicDirs, int partition, string topic, string consumerThreadId) { var partTopicInfoMap = currentTopicRegistry.Get(topic); var znode = topicDirs.ConsumerOffsetDir + "/" + partition; var offsetString = ZkUtils.ReadDataMaybeNull(parent.zkClient, znode).Item1; // If first time starting a consumer, set the initial offset to -1 var offset = (offsetString != null) ? long.Parse(offsetString) : PartitionTopicInfo.InvalidOffset; var queue = parent.topicThreadIdAndQueues.Get(Tuple.Create(topic, consumerThreadId)); var consumedOffset = new AtomicLong(offset); var fetchedOffset = new AtomicLong(offset); var partTopicInfo = new PartitionTopicInfo( topic, partition, queue, consumedOffset, fetchedOffset, new AtomicInteger(parent.Config.FetchMessageMaxBytes), parent.Config.ClientId); partTopicInfoMap[partition] = partTopicInfo; Logger.DebugFormat("{0} selected new offset {1}", partTopicInfo, offset); parent.checkpointedOffsets[new TopicAndPartition(topic, partition)] = offset; }
private void CreateFetchThread(PartitionTopicInfo partition, Broker broker) { if (_fetcherThreads == null) { return; } CreateFetchThread(new List <PartitionTopicInfo> { partition }, broker); }
/// <summary> /// Commit offset of specified topic/partition. /// Only used when customer has strong requirement for reprocess messages as few as possible. /// </summary> /// <param name="topic"></param> /// <param name="partition"></param> /// <param name="offset"></param> /// <param name="setPosition">Indicates whether to set the fetcher's offset to the value committed. Default = true.</param> public void CommitOffset(string topic, int partition, long offset, bool setPosition = true) { this.EnsuresNotDisposed(); if (this.GetZkClient() == null) { return; } if (this.config.AutoCommit == true) { throw new ArgumentException(string.Format("When do commit offset with desired partition and offset, must set AutoCommit of ConsumerConfiguration as false!")); } try { IDictionary <int, PartitionTopicInfo> topicPartitionInfo = topicRegistry[topic]; var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic); PartitionTopicInfo partitionTopicInfo = topicPartitionInfo[partition]; if (partitionTopicInfo.ConsumeOffsetValid) { //Commit offset unconditionally. This would give consumes to decide which offset to read/skip //if (offset > partitionTopicInfo.CommitedOffset) try { ZkUtils.UpdatePersistentPath(GetZkClient(), topicDirs.ConsumerOffsetDir + "/" + partitionTopicInfo.PartitionId, offset.ToString()); partitionTopicInfo.CommitedOffset = offset; if (setPosition) { partitionTopicInfo.ConsumeOffset = offset; partitionTopicInfo.FetchOffset = offset; } } catch (Exception ex) { Logger.ErrorFormat("error in CommitOffsets UpdatePersistentPath : {0}", ex.FormatException()); } } else { Logger.InfoFormat( "Skip committing offset {0} for topic {1} because it is invalid (ZK session is disconnected)", offset, partitionTopicInfo); } if (Logger.IsDebugEnabled) { Logger.DebugFormat("Commited offset {0} for topic {1}", offset, partitionTopicInfo); } } catch (Exception ex) { Logger.ErrorFormat("exception during CommitOffsets: Topic:{0} Partition:{1} offset:{2} Exception:{3} ", topic, partition, offset, ex.FormatException()); } }
private Message MakeNext() { if (current == null || !current.MoveNext()) { Logger.Debug("Getting new FetchedDataChunk..."); if (consumerTimeoutMs < 0) { currentDataChunk = this.channel.Take(cancellationToken); } else { bool done = channel.TryTake(out currentDataChunk, consumerTimeoutMs, cancellationToken); if (!done) { Logger.Debug("Consumer iterator timing out..."); state = ConsumerIteratorState.NotReady; throw new ConsumerTimeoutException(); } } if (currentDataChunk.Equals(ZookeeperConsumerConnector.ShutdownCommand)) { Logger.Debug("Received the shutdown command"); channel.Add(currentDataChunk); return(this.AllDone()); } currentTopicInfo = currentDataChunk.TopicInfo; Logger.DebugFormat("CurrentTopicInfo: ConsumedOffset({0}), FetchOffset({1})", currentTopicInfo.GetConsumeOffset(), currentTopicInfo.GetFetchOffset()); if (currentTopicInfo.GetConsumeOffset() != currentDataChunk.FetchOffset) { Logger.ErrorFormat( CultureInfo.CurrentCulture, "consumed offset: {0} doesn't match fetch offset: {1} for {2}; consumer may lose data", currentTopicInfo.GetConsumeOffset(), currentDataChunk.FetchOffset, currentTopicInfo); currentTopicInfo.ResetConsumeOffset(currentDataChunk.FetchOffset); } current = currentDataChunk.Messages.GetEnumerator(); current.MoveNext(); } var item = current.Current; consumedOffset = item.Offset; return(item.Message); }
public FetchedDataChunk(BufferedMessageSet messages, PartitionTopicInfo topicInfo, long fetchOffset) { this.Messages = messages; this.TopicInfo = topicInfo; this.FetchOffset = fetchOffset; }
internal void AddPartitionWithError(PartitionTopicInfo partition) { _partitionsNeedingLeaders.Enqueue(partition); }
protected override MessageAndMetadata <TKey, TValue> MakeNext() { FetchedDataChunk currentDataChunk; var localCurrent = this.current.Get(); if (localCurrent == null || !localCurrent.HasNext()) { if (this.consumerTimeoutMs < 0) { currentDataChunk = this.channel.Take(); } else { if (!this.channel.TryTake(out currentDataChunk, this.consumerTimeoutMs)) { // reste stat to make the iterator re-iterable this.ResetState(); throw new ConsumerTimeoutException(); } } if (currentDataChunk.Equals(ZookeeperConsumerConnector.ShutdownCommand)) { Logger.Debug("Received the shutdown command"); this.channel.Add(currentDataChunk); return(this.AllDone()); } else { this.currentTopicInfo = currentDataChunk.TopicInfo; var cdcFetchOffset = currentDataChunk.FetchOffset; var ctiConsumeOffset = this.currentTopicInfo.GetFetchOffset(); Logger.DebugFormat( "CurrentTopicInfo: ConsumedOffset({0}), FetchOffset({1})", this.currentTopicInfo.GetConsumeOffset(), this.currentTopicInfo.GetFetchOffset()); if (ctiConsumeOffset < cdcFetchOffset) { Logger.ErrorFormat( CultureInfo.CurrentCulture, "consumed offset: {0} doesn't match fetch offset: {1} for {2}; consumer may lose Data", ctiConsumeOffset, cdcFetchOffset, this.currentTopicInfo); this.currentTopicInfo.ResetConsumeOffset(currentDataChunk.FetchOffset); } localCurrent = currentDataChunk.Messages.Iterator(); this.current.Set(localCurrent); } // if we just updated the current chunk and it is empty that means the fetch size is too small! if (currentDataChunk.Messages.ValidBytes == 0) { throw new MessageSizeTooLargeException( string.Format( "Found a message larger than the maximum fetch size of this consumer on topic " + "{0} partition {1} at fetch offset {2}. Increase the fetch size, or decrease the maximum message size the broker will allow.", currentDataChunk.TopicInfo.Topic, currentDataChunk.TopicInfo.PartitionId, currentDataChunk.FetchOffset)); } } var item = localCurrent.Next(); // reject the messages that have already been consumed while (item.Offset < this.currentTopicInfo.GetConsumeOffset() && localCurrent.HasNext()) { item = localCurrent.Next(); } this.consumedOffset = item.NextOffset; item.Message.EnsureValid(); // validate checksum of message to ensure it is valid return(new MessageAndMetadata <TKey, TValue>( this.currentTopicInfo.Topic, this.currentTopicInfo.PartitionId, item.Message, item.Offset, this.keyDecoder, this.valueDecoder)); }
private void AddPartitionTopicInfo(ZKGroupTopicDirs topicDirs, string partitionString, string topic, string consumerThreadId) { var partition = Partition.ParseFrom(partitionString); var partTopicInfoMap = this.topicRegistry[topic]; var znode = topicDirs.ConsumerOffsetDir + "/" + partition.Name; var offsetString = this.zkClient.ReadData<string>(znode, true); long offset = string.IsNullOrEmpty(offsetString) ? 0 : long.Parse(offsetString, CultureInfo.InvariantCulture); var queue = this.queues[new Tuple<string, string>(topic, consumerThreadId)]; var partTopicInfo = new PartitionTopicInfo( topic, partition.BrokerId, partition, queue, offset, offset, this.config.FetchSize); partTopicInfoMap.Add(partition, partTopicInfo); if (Logger.IsDebugEnabled) { Logger.DebugFormat(CultureInfo.CurrentCulture, "{0} selected new offset {1}", partTopicInfo, offset); } }
/// <summary> /// Method to be used for starting a new thread /// </summary> internal void Run() { foreach (var partitionTopicInfo in partitionTopicInfos) { Logger.InfoFormat( CultureInfo.CurrentCulture, "{0} start fetching topic: {1} part: {2} offset: {3} from {4}:{5}", this.name, partitionTopicInfo.Topic, partitionTopicInfo.Partition.PartId, partitionTopicInfo.GetFetchOffset(), this.broker.Host, this.broker.Port); } try { while (!this.shouldStop) { var requestList = new List <FetchRequest>(); foreach (var partitionTopicInfo in this.partitionTopicInfos) { var singleRequest = new FetchRequest(partitionTopicInfo.Topic, partitionTopicInfo.Partition.PartId, partitionTopicInfo.GetFetchOffset(), this.config.MaxFetchSize); requestList.Add(singleRequest); } Logger.Debug("Fetch request: " + string.Join(", ", requestList.Select(x => x.ToString()))); var request = new MultiFetchRequest(requestList); var response = this.simpleConsumer.MultiFetch(request); int read = 0; var items = this.partitionTopicInfos.Zip( response, (x, y) => new Tuple <PartitionTopicInfo, BufferedMessageSet>(x, y)); foreach (Tuple <PartitionTopicInfo, BufferedMessageSet> item in items) { BufferedMessageSet messages = item.Item2; PartitionTopicInfo info = item.Item1; try { bool done = false; if (messages.ErrorCode == ErrorMapping.OffsetOutOfRangeCode) { Logger.InfoFormat(CultureInfo.CurrentCulture, "offset {0} out of range", info.GetFetchOffset()); //// see if we can fix this error var resetOffset = this.ResetConsumerOffsets(info.Topic, info.Partition); if (resetOffset >= 0) { info.ResetFetchOffset(resetOffset); info.ResetConsumeOffset(resetOffset); done = true; } } if (!done) { read += info.Add(messages, info.GetFetchOffset()); } } catch (Exception ex) { if (!shouldStop) { Logger.ErrorFormat(CultureInfo.CurrentCulture, "error in FetcherRunnable for {0}" + info, ex); } throw; } } Logger.Info("Fetched bytes: " + read); if (read == 0) { Logger.DebugFormat(CultureInfo.CurrentCulture, "backing off {0} ms", this.config.BackOffIncrement); Thread.Sleep(this.config.BackOffIncrement); } } } catch (Exception ex) { if (shouldStop) { Logger.InfoFormat(CultureInfo.CurrentCulture, "FetcherRunnable {0} interrupted", this); } else { Logger.ErrorFormat(CultureInfo.CurrentCulture, "error in FetcherRunnable {0}", ex); } } Logger.InfoFormat(CultureInfo.CurrentCulture, "stopping fetcher {0} to host {1}", this.name, this.broker.Host); }