private Message MakeNext() { if (current == null || !current.MoveNext()) { Logger.Debug("Getting new FetchedDataChunk..."); if (consumerTimeoutMs < 0) { currentDataChunk = this.channel.Take(cancellationToken); } else { bool done = channel.TryTake(out currentDataChunk, consumerTimeoutMs, cancellationToken); if (!done) { Logger.Debug("Consumer iterator timing out..."); state = ConsumerIteratorState.NotReady; throw new ConsumerTimeoutException(); } } if (currentDataChunk.Equals(ZookeeperConsumerConnector.ShutdownCommand)) { Logger.Debug("Received the shutdown command"); channel.Add(currentDataChunk); return(this.AllDone()); } currentTopicInfo = currentDataChunk.TopicInfo; Logger.DebugFormat("CurrentTopicInfo: ConsumedOffset({0}), FetchOffset({1})", currentTopicInfo.GetConsumeOffset(), currentTopicInfo.GetFetchOffset()); if (currentTopicInfo.GetConsumeOffset() != currentDataChunk.FetchOffset) { Logger.ErrorFormat( CultureInfo.CurrentCulture, "consumed offset: {0} doesn't match fetch offset: {1} for {2}; consumer may lose data", currentTopicInfo.GetConsumeOffset(), currentDataChunk.FetchOffset, currentTopicInfo); currentTopicInfo.ResetConsumeOffset(currentDataChunk.FetchOffset); } current = currentDataChunk.Messages.GetEnumerator(); current.MoveNext(); } var item = current.Current; consumedOffset = item.Offset; return(item.Message); }
private void SendShutdownToAllQueues() { foreach (var queue in this.queues) { Logger.InfoFormat("Clearing up queue"); // clear the queue while (queue.Value.Count > 0) { FetchedDataChunk item = null; queue.Value.TryTake(out item); } queue.Value.Add(ShutdownCommand); Logger.InfoFormat("Cleared queue and sent shutdown command"); } }
public void ClearIterator() { var semaphoreTaken = makeNextSemaphore.Wait(1000); try { while (channel.Count > 0) { FetchedDataChunk item = null; channel.TryTake(out item); } Logger.Info("Clearing the current data chunk for this consumer iterator"); current = null; } finally { if (semaphoreTaken) { makeNextSemaphore.Release(); } } }