public override void ProcessPartitionData( TopicAndPartition topicAndPartition, long fetchOffset, FetchResponsePartitionData partitionData) { var pti = this.partitionMap.Get(topicAndPartition); if (pti.GetFetchOffset() != fetchOffset) { throw new Exception(string.Format("Offset doesn't match for partition [{0},{1}] pti offset: {2} fetch offset: {3}", topicAndPartition.Topic, topicAndPartition.Partiton, pti.GetFetchOffset(), fetchOffset)); } pti.Enqueue((ByteBufferMessageSet)partitionData.Messages); }
public override long HandleOffsetOutOfRange(TopicAndPartition topicAndPartition) { long startTimestamp; switch (this.config.AutoOffsetReset) { case OffsetRequest.SmallestTimeString: startTimestamp = OffsetRequest.EarliestTime; break; case OffsetRequest.LargestTimeString: startTimestamp = OffsetRequest.LatestTime; break; default: startTimestamp = OffsetRequest.LatestTime; break; } var newOffset = simpleConsumer.EarliestOrLatestOffset( topicAndPartition, startTimestamp, Request.OrdinaryConsumerId); var pti = this.partitionMap.Get(topicAndPartition); pti.ResetFetchOffset(newOffset); pti.ResetConsumeOffset(newOffset); return newOffset; }
protected bool Equals(TopicAndPartition other) { return this.Topic == other.Topic && this.Partiton == other.Partiton; }
public override void DoWork() { var leaderForPartitionsMap = new Dictionary<TopicAndPartition, Broker>(); [email protected](); try { while (this.parent.NoLeaderPartitionSet.Count == 0) { Logger.Debug("No partition for leader election."); this.parent.cond.Await(); } Logger.DebugFormat("Partitions without leader {0}", string.Join(",", this.parent.NoLeaderPartitionSet)); var brokers = ZkUtils.GetAllBrokersInCluster(this.parent.zkClient); var topicsMetadata = ClientUtils.FetchTopicMetadata( new HashSet<string>(this.parent.NoLeaderPartitionSet.Select(m => m.Topic)), brokers, this.parent.config.ClientId, this.parent.config.SocketTimeoutMs, this.parent.correlationId.GetAndIncrement()).TopicsMetadata; if (Logger.IsDebugEnabled) { foreach (var topicMetadata in topicsMetadata) { Logger.Debug(topicMetadata); } } foreach (var tmd in topicsMetadata) { var topic = tmd.Topic; foreach (var pmd in tmd.PartitionsMetadata) { var topicAndPartition = new TopicAndPartition(topic, pmd.PartitionId); if (pmd.Leader != null && this.parent.NoLeaderPartitionSet.Contains(topicAndPartition)) { var leaderBroker = pmd.Leader; leaderForPartitionsMap[topicAndPartition] = leaderBroker; this.parent.NoLeaderPartitionSet.Remove(topicAndPartition); } } } } catch (Exception e) { if (!isRunning.Get()) { throw; /* If this thread is stopped, propagate this exception to kill the thread. */ } else { Logger.Warn("Failed to find leader for " + string.Join(",", this.parent.NoLeaderPartitionSet), e); } } finally { [email protected](); } try { this.parent.AddFetcherForPartitions( leaderForPartitionsMap.ToDictionary( kvp => kvp.Key, kvp => new BrokerAndInitialOffset(kvp.Value, this.parent.partitionMap.Get(kvp.Key).GetFetchOffset()))); } catch (Exception e) { if (!isRunning.Get()) { throw; /* If this thread is stopped, propagate this exception to kill the thread. */ } else { Logger.Warn(string.Format("Failed to add leader for partitions {0}; will retry", string.Join(",", leaderForPartitionsMap.Keys)), e); [email protected](); foreach (var leader in leaderForPartitionsMap.Keys) { this.parent.NoLeaderPartitionSet.Add(leader); } [email protected](); } } this.parent.ShutdownIdleFetcherThreads(); Thread.Sleep(this.parent.config.RefreshLeaderBackoffMs); }
/// <summary> /// Get the earliest or latest offset of a given topic, partition. /// </summary> /// <param name="topicAndPartition">Topic and partition of which the offset is needed.</param> /// <param name="earliestOrLatest">A value to indicate earliest or latest offset.</param> /// <param name="consumerId">Id of the consumer which could be a consumer client, SimpleConsumerShell or a follower broker.</param> /// <returns>Requested offset.</returns> public long EarliestOrLatestOffset(TopicAndPartition topicAndPartition, long earliestOrLatest, int consumerId) { var request = new OffsetRequest( new Dictionary<TopicAndPartition, PartitionOffsetRequestInfo> { { topicAndPartition, new PartitionOffsetRequestInfo(earliestOrLatest, 1) } }, clientId: this.ClientId, replicaId: consumerId); var partitionErrorAndOffset = this.GetOffsetsBefore(request).PartitionErrorAndOffsets[topicAndPartition]; long offset; if (partitionErrorAndOffset.Error == ErrorMapping.NoError) { offset = partitionErrorAndOffset.Offsets[0]; } else { throw ErrorMapping.ExceptionFor(partitionErrorAndOffset.Error); } return offset; }
/// <summary> /// process fetched Data /// </summary> /// <param name="topicAndPartition"></param> /// <param name="fetchOffset"></param> /// <param name="partitionData"></param> public abstract void ProcessPartitionData( TopicAndPartition topicAndPartition, long fetchOffset, FetchResponsePartitionData partitionData);
/// <summary> /// handle a partition whose offset is out of range and return a new fetch offset /// </summary> /// <param name="topicAndPartition"></param> /// <returns></returns> public abstract long HandleOffsetOutOfRange(TopicAndPartition topicAndPartition);
private FetchResponsePartitionData PartitionDataFor(string topic, int partition) { var topicAndPartition = new TopicAndPartition(topic, partition); FetchResponsePartitionData partitionData; if (this.Data.TryGetValue(topicAndPartition, out partitionData)) { return partitionData; } else { throw new ArgumentException(string.Format("No partition {0} in fetch response {1}", topicAndPartition, this)); } }
protected bool Equals(TopicAndPartition other) { return(this.Topic == other.Topic && this.Partiton == other.Partiton); }