/// <summary>
/// Selects a synchronous producer, for
/// the specified broker id and calls the send API on the selected
/// producer to publish the data to the specified broker partition.
/// </summary>
/// <param name="poolData">The producer pool request object.</param>
/// <remarks>
/// Used for multi-topic request
/// </remarks>
public override void Send(IEnumerable <ProducerPoolData <TData> > poolData)
{
this.EnsuresNotDisposed();
Guard.NotNull(poolData, "poolData");
Dictionary <int, List <ProducerPoolData <TData> > > distinctBrokers = poolData.GroupBy(
x => x.BidPid.BrokerId, x => x)
.ToDictionary(x => x.Key, x => x.ToList());
foreach (var broker in distinctBrokers)
{
Logger.DebugFormat(CultureInfo.CurrentCulture, "Fetching sync producer for broker id: {0}", broker.Key);
ISyncProducer producer = this.syncProducers[broker.Key];
IEnumerable <ProducerRequest> requests = broker.Value.Select(x => new ProducerRequest(
x.Topic,
x.BidPid.PartId,
new BufferedMessageSet(x.Data.Select(y => this.Serializer.ToMessage(y)))));
Logger.DebugFormat(CultureInfo.CurrentCulture, "Sending message to broker {0}", broker.Key);
if (requests.Count() > 1)
{
producer.MultiSend(requests);
}
else
{
producer.Send(requests.First());
}
}
}
public SyncProducerWrapper(ISyncProducer syncProducer, int count)
{
Producers = new Queue <ISyncProducer>(count);
for (int i = 0; i < count; i++)
{
Producers.Enqueue(syncProducer);
}
}
public ISyncProducer GetProducer()
{
//round-robin instead of random peek, to avoid collision of producers.
ISyncProducer producer = null;
lock (_lock)
{
producer = Producers.Peek();
Producers.Dequeue();
Producers.Enqueue(producer);
}
return(producer);
}
/// <summary>
/// Force get topic metadata and update
/// </summary>
public void UpdateInfo(short versionId, int correlationId, string clientId, string topic)
{
Logger.InfoFormat("Will update metadata for topic:{0}", topic);
Guard.NotNullNorEmpty(topic, "topic");
var shuffledBrokers = this.syncProducerPool.GetShuffledProducers();
var i = 0;
var hasFetchedInfo = false;
while (i < shuffledBrokers.Count && !hasFetchedInfo)
{
ISyncProducer producer = shuffledBrokers[i++];
try
{
var topicMetadataRequest = TopicMetadataRequest.Create(new List <string>()
{
topic
}, versionId,
correlationId, clientId);
var topicMetadataList = producer.Send(topicMetadataRequest);
var topicMetadata = topicMetadataList.Any() ? topicMetadataList.First() : null;
if (topicMetadata != null)
{
if (topicMetadata.Error != ErrorMapping.NoError)
{
Logger.WarnFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, topicMetadata.Error.ToString());
}
else
{
this.topicPartitionInfo[topic] = topicMetadata;
this.topicPartitionInfoLastUpdateTime[topic] = DateTime.UtcNow;
Logger.InfoFormat("Will Update metadata info, topic {0} ", topic);
//TODO: For all partitions which has metadata, here return the sorted list.
//But sometimes kafka didn't return metadata for all topics.
this.topicPartitionInfoList[topic] = topicMetadata.PartitionsMetadata.Select(m =>
{
Partition partition = new Partition(topic, m.PartitionId);
if (m.Leader != null)
{
var leaderReplica = new Replica(m.Leader.Id, topic);
partition.Leader = leaderReplica;
Logger.InfoFormat("Topic {0} partition {1} has leader {2}", topic,
m.PartitionId, m.Leader.Id);
return(partition);
}
Logger.WarnFormat("Topic {0} partition {1} does not have a leader yet", topic,
m.PartitionId);
return(partition);
}
).OrderBy(x => x.PartId).ToList();;
hasFetchedInfo = true;
Logger.InfoFormat("Finish Update metadata info, topic {0} Partitions:{1} No leader:{2}", topic, this.topicPartitionInfoList[topic].Count, this.topicPartitionInfoList[topic].Where(r => r.Leader == null).Count());
//In very weired case, the kafka broker didn't return metadata of all broker. need break and retry. https://issues.apache.org/jira/browse/KAFKA-1998
// http://qnalist.com/questions/5899394/topicmetadata-response-miss-some-partitions-information-sometimes
if (zkClient != null)
{
Dictionary <int, int[]> topicMetaDataInZookeeper = ZkUtils.GetTopicMetadataInzookeeper(this.zkClient, topic);
if (topicMetaDataInZookeeper != null && topicMetaDataInZookeeper.Any())
{
topicDataInZookeeper[topic] = topicMetaDataInZookeeper;
if (this.topicPartitionInfoList[topic].Count != topicMetaDataInZookeeper.Count)
{
Logger.ErrorFormat("NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count);
throw new UnavailableProducerException(string.Format("Please make sure every partition at least has one broker running and retry again. NOT all partition has metadata. Topic partition in zookeeper :{0} topics has partition metadata: {1}", topicMetaDataInZookeeper.Count, this.topicPartitionInfoList[topic].Count));
}
}
}
}
}
}
catch (Exception e)
{
Logger.ErrorFormat("Try get metadata of topic {0} from {1}({2}) . Got error: {3}", topic, producer.Config.BrokerId, producer.Config.Host, e.FormatException());
}
}
}
/// <summary>
/// Send message of one broker.
/// </summary>
/// <param name="brokerId"></param>
/// <param name="messagesPerTopic"></param>
/// <returns></returns>
private ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > > Send(int brokerId, IDictionary <TopicAndPartition, BufferedMessageSet> messagesPerTopic)
{
try
{
if (brokerId < 0)
{
throw new NoLeaderForPartitionException(
string.Format("No leader for some partition(s). And it try write to on invalid broker {0}. The assigned TopicAndPartition for the data is :{1} ", brokerId, messagesPerTopic.Any() ? messagesPerTopic.First().Key.ToString() : "(null)"));
}
if (messagesPerTopic.Any())
{
var producerRequest = new ProducerRequest(NextCorrelationId,
this.producerConfig.ClientId,
this.producerConfig.RequiredAcks,
this.producerConfig.AckTimeout,
messagesPerTopic);
ISyncProducer syncProducer = null;
try
{
syncProducer = this.syncProducerPool.GetProducer(brokerId);
}
catch (UnavailableProducerException e)
{
Logger.Error(e.Message);
// When initializing producer pool, some broker might be unavailable, and now it is healthy and is leader for some partitions.
// A new producer should be added to the pool, creating a TCP connection to the broker.
var broker =
this.brokerPartitionInfo.GetBrokerPartitionLeaders(messagesPerTopic.Keys.First().Topic)
.Values.FirstOrDefault(b => b.Id == brokerId);
if (broker != null)
{
this.syncProducerPool.AddProducer(broker);
syncProducer = this.syncProducerPool.GetProducer(brokerId);
}
}
if (producerConfig.Verbose)
{
Logger.DebugFormat("Kafka producer before sent messages for topics {0} to broker {1}", messagesPerTopic, brokerId);
}
ProducerResponse response = syncProducer.Send(producerRequest);
if (this.producerConfig.Verbose)
{
string msg = string.Format("Kafka producer sent messages for topics {0} to broker {1} on {2}:{3}",
messagesPerTopic, brokerId, syncProducer.Config.Host, syncProducer.Config.Port);
Logger.Debug(msg);
}
if (response != null)
{
int statusCount = response.Statuses.Count();
//In java version
//https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/producer/async/DefaultEventHandler.scala;h=821901e4f434dfd9eec6eceabfc2e1e65507a57c;hb=HEAD#l260
//The producerRequest.data just the messagesPerTopic. So there compare the statusCount with producerRequest.data.size
//But in this C# version, the producerRequest.Data already grouped by topic. So here need compare with messagesPerTopic.Count()
int requestCount = messagesPerTopic.Count();
if (statusCount != requestCount)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("Incomplete response count {0} for producer request count {1}. ", statusCount, requestCount);
sb.AppendFormat(" Broker {0} on {1}:{2}", brokerId, syncProducer.Config.Host, syncProducer.Config.Port);
sb.Append(" Message detail:");
sb.Append(string.Join(",", messagesPerTopic.Select(r => string.Format("{0},{1}", r.Key.Topic, r.Key.PartitionId))));
sb.Append(" Response status detail which has error:");
sb.Append(string.Join(",", response.Statuses.Where(r => r.Value.Error != (short)ErrorMapping.NoError).Select(r => r.ToString())));
throw new FailedToSendMessageException <TK>(sb.ToString());
}
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(response.Statuses.Where(s => s.Value.Error != (short)ErrorMapping.NoError)
.Select(s => new Tuple <TopicAndPartition, ProducerResponseStatus>(s.Key, s.Value))));
}
}
}
catch (NoLeaderForPartitionException e)
{
Logger.Error(ExceptionUtil.GetExceptionDetailInfo(e));
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(messagesPerTopic.Keys.Select(
s => new Tuple <TopicAndPartition, ProducerResponseStatus>(s, new ProducerResponseStatus {
Error = ErrorMapping.NotLeaderForPartitionCode
})), e));
}
catch (Exception e)
{
Logger.Error(ExceptionUtil.GetExceptionDetailInfo(e));
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(messagesPerTopic.Keys.Select(
s => new Tuple <TopicAndPartition, ProducerResponseStatus>(s, new ProducerResponseStatus {
Error = ErrorMapping.UnknownCode
})), e));
}
return(new ProducerSendResult <IEnumerable <Tuple <TopicAndPartition, ProducerResponseStatus> > >(Enumerable.Empty <Tuple <TopicAndPartition, ProducerResponseStatus> >()));
}
public SyncProducerWrapper(ISyncProducer syncProducer, int count)
{
Producers = new Queue<ISyncProducer>(count);
for (int i = 0; i < count; i++)
{
Producers.Enqueue(syncProducer);
}
}
