private Message MakeNext()
{
if (current == null || !current.MoveNext())
{
Logger.Debug("Getting new FetchedDataChunk...");
if (consumerTimeoutMs < 0)
{
currentDataChunk = this.channel.Take(cancellationToken);
}
else
{
bool done = channel.TryTake(out currentDataChunk, consumerTimeoutMs, cancellationToken);
if (!done)
{
Logger.Debug("Consumer iterator timing out...");
state = ConsumerIteratorState.NotReady;
throw new ConsumerTimeoutException();
}
}
if (currentDataChunk.Equals(ZookeeperConsumerConnector.ShutdownCommand))
{
Logger.Debug("Received the shutdown command");
channel.Add(currentDataChunk);
return(this.AllDone());
}
currentTopicInfo = currentDataChunk.TopicInfo;
Logger.DebugFormat("CurrentTopicInfo: ConsumedOffset({0}), FetchOffset({1})",
currentTopicInfo.GetConsumeOffset(), currentTopicInfo.GetFetchOffset());
if (currentTopicInfo.GetConsumeOffset() != currentDataChunk.FetchOffset)
{
Logger.ErrorFormat(
CultureInfo.CurrentCulture,
"consumed offset: {0} doesn't match fetch offset: {1} for {2}; consumer may lose data",
currentTopicInfo.GetConsumeOffset(),
currentDataChunk.FetchOffset,
currentTopicInfo);
currentTopicInfo.ResetConsumeOffset(currentDataChunk.FetchOffset);
}
current = currentDataChunk.Messages.GetEnumerator();
current.MoveNext();
}
var item = current.Current;
consumedOffset = item.Offset;
return(item.Message);
}
private void SendShutdownToAllQueues()
{
foreach (var queue in this.queues)
{
Logger.InfoFormat("Clearing up queue");
// clear the queue
while (queue.Value.Count > 0)
{
FetchedDataChunk item = null;
queue.Value.TryTake(out item);
}
queue.Value.Add(ShutdownCommand);
Logger.InfoFormat("Cleared queue and sent shutdown command");
}
}
public void ClearIterator()
{
var semaphoreTaken = makeNextSemaphore.Wait(1000);
try
{
while (channel.Count > 0)
{
FetchedDataChunk item = null;
channel.TryTake(out item);
}
Logger.Info("Clearing the current data chunk for this consumer iterator");
current = null;
}
finally
{
if (semaphoreTaken)
{
makeNextSemaphore.Release();
}
}
}
