| public ResetConsumeOffset ( long newConsumeOffset ) : void | ||
| newConsumeOffset | long | |
| Résultat | void | |
private Message MakeNext()
{
if (current == null || !current.MoveNext())
{
Logger.Debug("Getting new FetchedDataChunk...");
if (consumerTimeoutMs < 0)
{
currentDataChunk = this.channel.Take(cancellationToken);
}
else
{
bool done = channel.TryTake(out currentDataChunk, consumerTimeoutMs, cancellationToken);
if (!done)
{
Logger.Debug("Consumer iterator timing out...");
state = ConsumerIteratorState.NotReady;
throw new ConsumerTimeoutException();
}
}
if (currentDataChunk.Equals(ZookeeperConsumerConnector.ShutdownCommand))
{
Logger.Debug("Received the shutdown command");
channel.Add(currentDataChunk);
return(this.AllDone());
}
currentTopicInfo = currentDataChunk.TopicInfo;
Logger.DebugFormat("CurrentTopicInfo: ConsumedOffset({0}), FetchOffset({1})",
currentTopicInfo.GetConsumeOffset(), currentTopicInfo.GetFetchOffset());
if (currentTopicInfo.GetConsumeOffset() != currentDataChunk.FetchOffset)
{
Logger.ErrorFormat(
CultureInfo.CurrentCulture,
"consumed offset: {0} doesn't match fetch offset: {1} for {2}; consumer may lose data",
currentTopicInfo.GetConsumeOffset(),
currentDataChunk.FetchOffset,
currentTopicInfo);
currentTopicInfo.ResetConsumeOffset(currentDataChunk.FetchOffset);
}
current = currentDataChunk.Messages.GetEnumerator();
current.MoveNext();
}
var item = current.Current;
consumedOffset = item.Offset;
return(item.Message);
}
/// <summary>
/// Method to be used for starting a new thread
/// </summary>
internal void Run()
{
foreach (var partitionTopicInfo in partitionTopicInfos)
{
Logger.InfoFormat(
CultureInfo.CurrentCulture,
"{0} start fetching topic: {1} part: {2} offset: {3} from {4}:{5}",
this.name,
partitionTopicInfo.Topic,
partitionTopicInfo.Partition.PartId,
partitionTopicInfo.GetFetchOffset(),
this.broker.Host,
this.broker.Port);
}
try
{
while (!this.shouldStop)
{
var requestList = new List <FetchRequest>();
foreach (var partitionTopicInfo in this.partitionTopicInfos)
{
var singleRequest = new FetchRequest(partitionTopicInfo.Topic, partitionTopicInfo.Partition.PartId, partitionTopicInfo.GetFetchOffset(), this.config.MaxFetchSize);
requestList.Add(singleRequest);
}
Logger.Debug("Fetch request: " + string.Join(", ", requestList.Select(x => x.ToString())));
var request = new MultiFetchRequest(requestList);
var response = this.simpleConsumer.MultiFetch(request);
int read = 0;
var items = this.partitionTopicInfos.Zip(
response,
(x, y) =>
new Tuple <PartitionTopicInfo, BufferedMessageSet>(x, y));
foreach (Tuple <PartitionTopicInfo, BufferedMessageSet> item in items)
{
BufferedMessageSet messages = item.Item2;
PartitionTopicInfo info = item.Item1;
try
{
bool done = false;
if (messages.ErrorCode == ErrorMapping.OffsetOutOfRangeCode)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "offset {0} out of range", info.GetFetchOffset());
//// see if we can fix this error
var resetOffset = this.ResetConsumerOffsets(info.Topic, info.Partition);
if (resetOffset >= 0)
{
info.ResetFetchOffset(resetOffset);
info.ResetConsumeOffset(resetOffset);
done = true;
}
}
if (!done)
{
read += info.Add(messages, info.GetFetchOffset());
}
}
catch (Exception ex)
{
if (!shouldStop)
{
Logger.ErrorFormat(CultureInfo.CurrentCulture, "error in FetcherRunnable for {0}" + info, ex);
}
throw;
}
}
Logger.Info("Fetched bytes: " + read);
if (read == 0)
{
Logger.DebugFormat(CultureInfo.CurrentCulture, "backing off {0} ms", this.config.BackOffIncrement);
Thread.Sleep(this.config.BackOffIncrement);
}
}
}
catch (Exception ex)
{
if (shouldStop)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "FetcherRunnable {0} interrupted", this);
}
else
{
Logger.ErrorFormat(CultureInfo.CurrentCulture, "error in FetcherRunnable {0}", ex);
}
}
Logger.InfoFormat(CultureInfo.CurrentCulture, "stopping fetcher {0} to host {1}", this.name, this.broker.Host);
}
