public override void ProcessPartitionData(
TopicAndPartition topicAndPartition, long fetchOffset, FetchResponsePartitionData partitionData)
{
var pti = this.partitionMap.Get(topicAndPartition);
if (pti.GetFetchOffset() != fetchOffset)
{
throw new Exception(string.Format("Offset doesn't match for partition [{0},{1}] pti offset: {2} fetch offset: {3}", topicAndPartition.Topic, topicAndPartition.Partiton, pti.GetFetchOffset(), fetchOffset));
}
pti.Enqueue((ByteBufferMessageSet)partitionData.Messages);
}
public override long HandleOffsetOutOfRange(TopicAndPartition topicAndPartition)
{
long startTimestamp;
switch (this.config.AutoOffsetReset)
{
case OffsetRequest.SmallestTimeString:
startTimestamp = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTimeString:
startTimestamp = OffsetRequest.LatestTime;
break;
default:
startTimestamp = OffsetRequest.LatestTime;
break;
}
var newOffset = simpleConsumer.EarliestOrLatestOffset(
topicAndPartition, startTimestamp, Request.OrdinaryConsumerId);
var pti = this.partitionMap.Get(topicAndPartition);
pti.ResetFetchOffset(newOffset);
pti.ResetConsumeOffset(newOffset);
return newOffset;
}
protected bool Equals(TopicAndPartition other)
{
return this.Topic == other.Topic && this.Partiton == other.Partiton;
}
public override void DoWork()
{
var leaderForPartitionsMap = new Dictionary<TopicAndPartition, Broker>();
[email protected]();
try
{
while (this.parent.NoLeaderPartitionSet.Count == 0)
{
Logger.Debug("No partition for leader election.");
this.parent.cond.Await();
}
Logger.DebugFormat("Partitions without leader {0}", string.Join(",", this.parent.NoLeaderPartitionSet));
var brokers = ZkUtils.GetAllBrokersInCluster(this.parent.zkClient);
var topicsMetadata =
ClientUtils.FetchTopicMetadata(
new HashSet<string>(this.parent.NoLeaderPartitionSet.Select(m => m.Topic)),
brokers,
this.parent.config.ClientId,
this.parent.config.SocketTimeoutMs,
this.parent.correlationId.GetAndIncrement()).TopicsMetadata;
if (Logger.IsDebugEnabled)
{
foreach (var topicMetadata in topicsMetadata)
{
Logger.Debug(topicMetadata);
}
}
foreach (var tmd in topicsMetadata)
{
var topic = tmd.Topic;
foreach (var pmd in tmd.PartitionsMetadata)
{
var topicAndPartition = new TopicAndPartition(topic, pmd.PartitionId);
if (pmd.Leader != null && this.parent.NoLeaderPartitionSet.Contains(topicAndPartition))
{
var leaderBroker = pmd.Leader;
leaderForPartitionsMap[topicAndPartition] = leaderBroker;
this.parent.NoLeaderPartitionSet.Remove(topicAndPartition);
}
}
}
}
catch (Exception e)
{
if (!isRunning.Get())
{
throw; /* If this thread is stopped, propagate this exception to kill the thread. */
}
else
{
Logger.Warn("Failed to find leader for " + string.Join(",", this.parent.NoLeaderPartitionSet), e);
}
}
finally
{
[email protected]();
}
try
{
this.parent.AddFetcherForPartitions(
leaderForPartitionsMap.ToDictionary(
kvp => kvp.Key,
kvp =>
new BrokerAndInitialOffset(kvp.Value, this.parent.partitionMap.Get(kvp.Key).GetFetchOffset())));
}
catch (Exception e)
{
if (!isRunning.Get())
{
throw; /* If this thread is stopped, propagate this exception to kill the thread. */
}
else
{
Logger.Warn(string.Format("Failed to add leader for partitions {0}; will retry", string.Join(",", leaderForPartitionsMap.Keys)), e);
[email protected]();
foreach (var leader in leaderForPartitionsMap.Keys)
{
this.parent.NoLeaderPartitionSet.Add(leader);
}
[email protected]();
}
}
this.parent.ShutdownIdleFetcherThreads();
Thread.Sleep(this.parent.config.RefreshLeaderBackoffMs);
}
/// <summary>
/// Get the earliest or latest offset of a given topic, partition.
/// </summary>
/// <param name="topicAndPartition">Topic and partition of which the offset is needed.</param>
/// <param name="earliestOrLatest">A value to indicate earliest or latest offset.</param>
/// <param name="consumerId">Id of the consumer which could be a consumer client, SimpleConsumerShell or a follower broker.</param>
/// <returns>Requested offset.</returns>
public long EarliestOrLatestOffset(TopicAndPartition topicAndPartition, long earliestOrLatest, int consumerId)
{
var request =
new OffsetRequest(
new Dictionary<TopicAndPartition, PartitionOffsetRequestInfo>
{
{
topicAndPartition,
new PartitionOffsetRequestInfo(earliestOrLatest, 1)
}
},
clientId: this.ClientId,
replicaId: consumerId);
var partitionErrorAndOffset = this.GetOffsetsBefore(request).PartitionErrorAndOffsets[topicAndPartition];
long offset;
if (partitionErrorAndOffset.Error == ErrorMapping.NoError)
{
offset = partitionErrorAndOffset.Offsets[0];
}
else
{
throw ErrorMapping.ExceptionFor(partitionErrorAndOffset.Error);
}
return offset;
}
/// <summary>
/// process fetched Data
/// </summary>
/// <param name="topicAndPartition"></param>
/// <param name="fetchOffset"></param>
/// <param name="partitionData"></param>
public abstract void ProcessPartitionData(
TopicAndPartition topicAndPartition, long fetchOffset, FetchResponsePartitionData partitionData);
/// <summary> /// handle a partition whose offset is out of range and return a new fetch offset /// </summary> /// <param name="topicAndPartition"></param> /// <returns></returns> public abstract long HandleOffsetOutOfRange(TopicAndPartition topicAndPartition);
private FetchResponsePartitionData PartitionDataFor(string topic, int partition)
{
var topicAndPartition = new TopicAndPartition(topic, partition);
FetchResponsePartitionData partitionData;
if (this.Data.TryGetValue(topicAndPartition, out partitionData))
{
return partitionData;
}
else
{
throw new ArgumentException(string.Format("No partition {0} in fetch response {1}", topicAndPartition, this));
}
}
protected bool Equals(TopicAndPartition other)
{
return(this.Topic == other.Topic && this.Partiton == other.Partiton);
}