public KafkaClientHelper(string topicParam, int partitionIndex, KafkaClientHelperConfiguration helperConfiguration)
{
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
this.partitionIndex = partitionIndex;
this.helperConfiguration = helperConfiguration;
this.MaxMessageSize = helperConfiguration.MaxMessageSize;
this.MaxMessagePerSet = helperConfiguration.MaxMessagePerSet;
this.ConsumerMaxWait = helperConfiguration.MaxWaitTime;
this.Offset = helperConfiguration.Offset;
this.OffsetType = helperConfiguration.OffsetType;
// Logger.Debug(string.Format("KafkaClientHelper constructor start,topicParam={0},partitionIndex={1},kafkaBrokerList={2}", topicParam, partitionIndex, kafkaBrokerList));
if (string.IsNullOrEmpty(this.helperConfiguration.KafkaBrokerList))
{
throw new System.ArgumentNullException("kafkaBrokerList");
}
if (string.IsNullOrEmpty(topicParam))
{
throw new System.ArgumentNullException("topicParam");
}
this.topic = topicParam;
// assemble brokerConfig list
this.brokerConfList = new List<BrokerConfiguration>(5);
string[] brokers = this.helperConfiguration.KafkaBrokerList.Split(new char[] { ',' });
int i = 1;
foreach (string v in brokers)
{
string[] brokerParams = v.Split(new char[] { ':' });
int port = 0;
if (brokerParams.Count() == 2
&& !string.IsNullOrEmpty(brokerParams[0])
&& int.TryParse(brokerParams[1], out port))
{
this.brokerConfList.Add(new BrokerConfiguration() { BrokerId = i, Host = brokerParams[0], Port = port });
}
i++;
}
// prepare SyncProducer list
i = 0;
syncProducerPool = new SyncProducer[this.brokerConfList.Count];
this.RoundRobinSyncProducer();
topicMetaRequest = TopicMetadataRequest.Create(
new string[] { this.topic }, // topic
0, // API version id
this.random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id
}
public KafkaClientHelper(string topicParam, int partitionIndex, string kafkaBrokerList, KafkaClientHelperConfiguration helperConfiguration)
: this(topicParam,partitionIndex,helperConfiguration)
{
if (!string.IsNullOrEmpty(kafkaBrokerList)
&& !string.IsNullOrEmpty(helperConfiguration.KafkaBrokerList)
&& kafkaBrokerList.ToUpperInvariant() != helperConfiguration.KafkaBrokerList.ToUpperInvariant())
{
throw new ArgumentException(string.Format("KafkaBrokerList doesn't match, please double check. {0} != {1}", kafkaBrokerList, helperConfiguration.KafkaBrokerList));
}
}
public KafkaClientHelper(string topicParam, int partitionIndex, KafkaClientHelperConfiguration helperConfiguration)
{
ServicePointManager.DefaultConnectionLimit = 5000;
ServicePointManager.UseNagleAlgorithm = false;
this.partitionIndex = partitionIndex;
this.helperConfiguration = helperConfiguration;
this.MaxMessageSize = helperConfiguration.MaxMessageSize;
this.MaxMessagePerSet = helperConfiguration.MaxMessagePerSet;
this.ConsumerMaxWait = helperConfiguration.MaxWaitTime;
this.Offset = helperConfiguration.Offset;
this.OffsetType = helperConfiguration.OffsetType;
// Logger.Debug(string.Format("KafkaClientHelper constructor start,topicParam={0},partitionIndex={1},kafkaBrokerList={2}", topicParam, partitionIndex, kafkaBrokerList));
if (string.IsNullOrEmpty(this.helperConfiguration.KafkaBrokerList))
{
throw new System.ArgumentNullException("kafkaBrokerList");
}
if (string.IsNullOrEmpty(topicParam))
{
throw new System.ArgumentNullException("topicParam");
}
this.topic = topicParam;
// assemble brokerConfig list
this.brokerConfList = new List <BrokerConfiguration>(5);
string[] brokers = this.helperConfiguration.KafkaBrokerList.Split(new char[] { ',' });
int i = 1;
foreach (string v in brokers)
{
string[] brokerParams = v.Split(new char[] { ':' });
int port = 0;
if (brokerParams.Count() == 2 &&
!string.IsNullOrEmpty(brokerParams[0]) &&
int.TryParse(brokerParams[1], out port))
{
this.brokerConfList.Add(new BrokerConfiguration()
{
BrokerId = i, Host = brokerParams[0], Port = port
});
}
i++;
}
// prepare SyncProducer list
i = 0;
syncProducerPool = new SyncProducer[this.brokerConfList.Count];
this.RoundRobinSyncProducer();
topicMetaRequest = TopicMetadataRequest.Create(
new string[] { this.topic }, // topic
0, // API version id
this.random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id
}
public KafkaClientHelper(string topicParam, int partitionIndex, string kafkaBrokerList, KafkaClientHelperConfiguration helperConfiguration)
: this(topicParam, partitionIndex, helperConfiguration)
{
if (!string.IsNullOrEmpty(kafkaBrokerList) &&
!string.IsNullOrEmpty(helperConfiguration.KafkaBrokerList) &&
kafkaBrokerList.ToUpperInvariant() != helperConfiguration.KafkaBrokerList.ToUpperInvariant())
{
throw new ArgumentException(string.Format("KafkaBrokerList doesn't match, please double check. {0} != {1}", kafkaBrokerList, helperConfiguration.KafkaBrokerList));
}
}
public static PullResponse PullMessage(
ConsumerConfiguration consumerConfig,
BrokerConfiguration leaderBrokerConfig,
int correlationID,
string topic,
int partitionIndex,
long fetchOffset,
KafkaClientHelperConfiguration helperConfiguration,
out long offsetNew)
{
offsetNew = -1;
PullResponse result = null;
int payloadCount = 0;
// at least retry once
int maxRetry = 1;
int retryCount = 0;
string s = string.Empty;
bool success = false;
while (!success && retryCount < maxRetry)
{
try
{
var requestMap = new Dictionary<string, List<PartitionFetchInfo>>();
requestMap.Add(
topic,
new List<PartitionFetchInfo>()
{
new PartitionFetchInfo(
partitionIndex,
fetchOffset,
helperConfiguration.FetchSize)
});
using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port))
{
var response = consumer.Fetch(new FetchRequest(
correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id
helperConfiguration.MaxWaitTime,
helperConfiguration.MinWaitBytes,
requestMap));
if (response == null)
{
throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
var partitionData = response.PartitionData(topic, partitionIndex);
if (partitionData == null)
{
throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
{
s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return null;
}
if (partitionData.Error != ErrorMapping.NoError)
{
s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return null;
}
var messages = partitionData.MessageSet.Messages;
s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
success = true;
result = new PullResponse(partitionData);
if (null != messages && messages.Count() > 0)
{
payloadCount = messages.Count();
long lastOffset = messages.Last().Offset;
if ((payloadCount + fetchOffset) != (lastOffset + 1))
{
s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry);
}
offsetNew = messages.Last().Offset + 1;
}
return result;
}
}
catch (Exception)
{
if (retryCount >= maxRetry)
{
throw;
}
}
finally
{
retryCount++;
}
} // end of while loop
return result;
}
