private Message MakeNext() { if (current == null || !current.MoveNext()) { Logger.Debug("Getting new FetchedDataChunk..."); if (consumerTimeoutMs < 0) { currentDataChunk = this.channel.Take(cancellationToken); } else { bool done = channel.TryTake(out currentDataChunk, consumerTimeoutMs, cancellationToken); if (!done) { Logger.Debug("Consumer iterator timing out..."); state = ConsumerIteratorState.NotReady; throw new ConsumerTimeoutException(); } } if (currentDataChunk.Equals(ZookeeperConsumerConnector.ShutdownCommand)) { Logger.Debug("Received the shutdown command"); channel.Add(currentDataChunk); return(this.AllDone()); } currentTopicInfo = currentDataChunk.TopicInfo; Logger.DebugFormat("CurrentTopicInfo: ConsumedOffset({0}), FetchOffset({1})", currentTopicInfo.GetConsumeOffset(), currentTopicInfo.GetFetchOffset()); if (currentTopicInfo.GetConsumeOffset() != currentDataChunk.FetchOffset) { Logger.ErrorFormat( CultureInfo.CurrentCulture, "consumed offset: {0} doesn't match fetch offset: {1} for {2}; consumer may lose data", currentTopicInfo.GetConsumeOffset(), currentDataChunk.FetchOffset, currentTopicInfo); currentTopicInfo.ResetConsumeOffset(currentDataChunk.FetchOffset); } current = currentDataChunk.Messages.GetEnumerator(); current.MoveNext(); } var item = current.Current; consumedOffset = item.Offset; return(item.Message); }
/// <summary> /// Method to be used for starting a new thread /// </summary> internal void Run() { foreach (var partitionTopicInfo in partitionTopicInfos) { Logger.InfoFormat( CultureInfo.CurrentCulture, "{0} start fetching topic: {1} part: {2} offset: {3} from {4}:{5}", this.name, partitionTopicInfo.Topic, partitionTopicInfo.Partition.PartId, partitionTopicInfo.GetFetchOffset(), this.broker.Host, this.broker.Port); } try { while (!this.shouldStop) { var requestList = new List <FetchRequest>(); foreach (var partitionTopicInfo in this.partitionTopicInfos) { var singleRequest = new FetchRequest(partitionTopicInfo.Topic, partitionTopicInfo.Partition.PartId, partitionTopicInfo.GetFetchOffset(), this.config.MaxFetchSize); requestList.Add(singleRequest); } Logger.Debug("Fetch request: " + string.Join(", ", requestList.Select(x => x.ToString()))); var request = new MultiFetchRequest(requestList); var response = this.simpleConsumer.MultiFetch(request); int read = 0; var items = this.partitionTopicInfos.Zip( response, (x, y) => new Tuple <PartitionTopicInfo, BufferedMessageSet>(x, y)); foreach (Tuple <PartitionTopicInfo, BufferedMessageSet> item in items) { BufferedMessageSet messages = item.Item2; PartitionTopicInfo info = item.Item1; try { bool done = false; if (messages.ErrorCode == ErrorMapping.OffsetOutOfRangeCode) { Logger.InfoFormat(CultureInfo.CurrentCulture, "offset {0} out of range", info.GetFetchOffset()); //// see if we can fix this error var resetOffset = this.ResetConsumerOffsets(info.Topic, info.Partition); if (resetOffset >= 0) { info.ResetFetchOffset(resetOffset); info.ResetConsumeOffset(resetOffset); done = true; } } if (!done) { read += info.Add(messages, info.GetFetchOffset()); } } catch (Exception ex) { if (!shouldStop) { Logger.ErrorFormat(CultureInfo.CurrentCulture, "error in FetcherRunnable for {0}" + info, ex); } throw; } } Logger.Info("Fetched bytes: " + read); if (read == 0) { Logger.DebugFormat(CultureInfo.CurrentCulture, "backing off {0} ms", this.config.BackOffIncrement); Thread.Sleep(this.config.BackOffIncrement); } } } catch (Exception ex) { if (shouldStop) { Logger.InfoFormat(CultureInfo.CurrentCulture, "FetcherRunnable {0} interrupted", this); } else { Logger.ErrorFormat(CultureInfo.CurrentCulture, "error in FetcherRunnable {0}", ex); } } Logger.InfoFormat(CultureInfo.CurrentCulture, "stopping fetcher {0} to host {1}", this.name, this.broker.Host); }