public ConsumerConfiguration(string host, int port)
: this()
{
Broker = new BrokerConfiguration {
Host = host, Port = port
};
}
private void SetBrokerConfiguration(BrokerConfigurationElement config)
{
this.Broker = new BrokerConfiguration
{
BrokerId = config.Id,
Host = GetIpAddress(config.Host),
Port = config.Port
};
}
public static string GetBrokerConfiturationString(int partitionIndex, BrokerConfiguration broker, bool isleader)
{
var sb = new StringBuilder();
if (isleader)
sb.AppendFormat("partition={0},leader[{0}]={1}", partitionIndex, broker);
else
sb.AppendFormat("partition={0},nonleader[{0}]={1}", partitionIndex, broker);
return sb.ToString();
}
public bool Equals(BrokerConfiguration p)
{
// If parameter is null return false:
if ((object)p == null)
{
return(false);
}
// Return true if the fields match:
return((BrokerId == p.BrokerId) && (Host == p.Host) && (Port == p.Port));
}
public bool Equals(BrokerConfiguration p)
{
// If parameter is null return false:
if (p == null)
{
return(false);
}
// Return true if the fields match:
return(BrokerId == p.BrokerId && Host == p.Host && Port == p.Port);
}
public bool Equals(BrokerConfiguration p)
{
// If parameter is null return false:
if ((object)p == null)
{
return false;
}
// Return true if the fields match:
return (BrokerId == p.BrokerId) && (Host == p.Host) && (Port == p.Port);
}
public static string GetBrokerConfiturationString(int partitionIndex, BrokerConfiguration broker, bool isleader)
{
var sb = new StringBuilder();
if (isleader)
{
sb.AppendFormat("partition={0},leader[{0}]={1}", partitionIndex, broker);
}
else
{
sb.AppendFormat("partition={0},nonleader[{0}]={1}", partitionIndex, broker);
}
return(sb.ToString());
}
public override bool Equals(System.Object obj)
{
// If parameter is null return false.
if (obj == null)
{
return(false);
}
// If parameter cannot be cast to Point return false.
BrokerConfiguration p = obj as BrokerConfiguration;
if ((System.Object)p == null)
{
return(false);
}
// Return true if the fields match:
return((BrokerId == p.BrokerId) && (Host == p.Host) && (Port == p.Port));
}
public ConsumerConfiguration(ConsumerConfiguration cosumerConfigTemplate, BrokerConfiguration brokerConfiguration)
{
this.Broker = brokerConfiguration;
this.NumberOfTries = cosumerConfigTemplate.NumberOfTries;
this.Timeout = cosumerConfigTemplate.Timeout;
this.AutoOffsetReset = cosumerConfigTemplate.AutoOffsetReset;
this.AutoCommit = cosumerConfigTemplate.AutoCommit;
this.AutoCommitInterval = cosumerConfigTemplate.AutoCommitInterval;
this.FetchSize = cosumerConfigTemplate.FetchSize;
this.MaxFetchFactor = cosumerConfigTemplate.MaxFetchFactor;
this.BackOffIncrement = cosumerConfigTemplate.BackOffIncrement;
this.ConsumerId = cosumerConfigTemplate.ConsumerId;
this.ReconnectInterval = cosumerConfigTemplate.ReconnectInterval;
this.ShutdownTimeout = cosumerConfigTemplate.ShutdownTimeout;
this.MaxFetchBufferLength = cosumerConfigTemplate.MaxFetchBufferLength;
this.SendTimeout = cosumerConfigTemplate.SendTimeout;
this.ReceiveTimeout = cosumerConfigTemplate.ReceiveTimeout;
this.BufferSize = cosumerConfigTemplate.BufferSize;
this.Verbose = cosumerConfigTemplate.Verbose;
this.ConsumeGroupRebalanceRetryIntervalMs = cosumerConfigTemplate.ConsumeGroupRebalanceRetryIntervalMs;
this.ConsumeGroupFindNewLeaderSleepIntervalMs = cosumerConfigTemplate.ConsumeGroupFindNewLeaderSleepIntervalMs;
}
/// <summary>
/// Convert a broker string into a BrokerConfiguration.
/// </summary>
/// <param name="brokerAddr">Broker string must follow the format of {Broker-ID},{Host}:{Port}</param>
/// <returns></returns>
public static BrokerConfiguration ToBrokerConfig(string brokerAddr)
{
if (string.IsNullOrEmpty(brokerAddr))
{
return null;
}
string[] brokerParams = brokerAddr.Split(',');
if (brokerParams == null || brokerParams.Length != 2)
{
return null;
}
int brokerId = -1;
if (!int.TryParse(brokerParams[0], out brokerId))
{
return null;
}
string[] hostParams = brokerParams[1].Split(':');
if (hostParams == null || hostParams.Length != 2)
{
return null;
}
int portNumber = -1;
if (!int.TryParse(hostParams[1], out portNumber))
{
return null;
}
var broker = new BrokerConfiguration() { BrokerId = brokerId, Host = hostParams[0], Port = portNumber };
return broker;
}
public static PullResponse PullMessage(
ConsumerConfiguration consumerConfig,
BrokerConfiguration leaderBrokerConfig,
int correlationID,
string topic,
int partitionIndex,
long fetchOffset,
KafkaClientHelperConfiguration helperConfiguration,
out long offsetNew)
{
offsetNew = -1;
PullResponse result = null;
int payloadCount = 0;
// at least retry once
int maxRetry = 1;
int retryCount = 0;
string s = string.Empty;
bool success = false;
while (!success && retryCount < maxRetry)
{
try
{
var requestMap = new Dictionary<string, List<PartitionFetchInfo>>();
requestMap.Add(
topic,
new List<PartitionFetchInfo>()
{
new PartitionFetchInfo(
partitionIndex,
fetchOffset,
helperConfiguration.FetchSize)
});
using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port))
{
var response = consumer.Fetch(new FetchRequest(
correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id
helperConfiguration.MaxWaitTime,
helperConfiguration.MinWaitBytes,
requestMap));
if (response == null)
{
throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
var partitionData = response.PartitionData(topic, partitionIndex);
if (partitionData == null)
{
throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
{
s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return null;
}
if (partitionData.Error != ErrorMapping.NoError)
{
s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return null;
}
var messages = partitionData.MessageSet.Messages;
s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
success = true;
result = new PullResponse(partitionData);
if (null != messages && messages.Count() > 0)
{
payloadCount = messages.Count();
long lastOffset = messages.Last().Offset;
if ((payloadCount + fetchOffset) != (lastOffset + 1))
{
s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry);
}
offsetNew = messages.Last().Offset + 1;
}
return result;
}
}
catch (Exception)
{
if (retryCount >= maxRetry)
{
throw;
}
}
finally
{
retryCount++;
}
} // end of while loop
return result;
}
public ConsumerConfiguration(ConsumerConfiguration cosumerConfigTemplate, BrokerConfiguration brokerConfiguration)
{
this.Broker = brokerConfiguration;
this.NumberOfTries = cosumerConfigTemplate.NumberOfTries;
this.Timeout = cosumerConfigTemplate.Timeout;
this.AutoOffsetReset = cosumerConfigTemplate.AutoOffsetReset;
this.AutoCommit = cosumerConfigTemplate.AutoCommit;
this.AutoCommitInterval = cosumerConfigTemplate.AutoCommitInterval;
this.FetchSize = cosumerConfigTemplate.FetchSize;
this.MaxFetchFactor = cosumerConfigTemplate.MaxFetchFactor;
this.BackOffIncrement = cosumerConfigTemplate.BackOffIncrement;
this.ConsumerId = cosumerConfigTemplate.ConsumerId;
this.ReconnectInterval = cosumerConfigTemplate.ReconnectInterval;
this.ShutdownTimeout = cosumerConfigTemplate.ShutdownTimeout;
this.MaxFetchBufferLength = cosumerConfigTemplate.MaxFetchBufferLength;
this.SendTimeout = cosumerConfigTemplate.SendTimeout;
this.ReceiveTimeout = cosumerConfigTemplate.ReceiveTimeout;
this.BufferSize = cosumerConfigTemplate.BufferSize;
this.Verbose = cosumerConfigTemplate.Verbose;
this.ConsumeGroupRebalanceRetryIntervalMs = cosumerConfigTemplate.ConsumeGroupRebalanceRetryIntervalMs;
this.ConsumeGroupFindNewLeaderSleepIntervalMs = cosumerConfigTemplate.ConsumeGroupFindNewLeaderSleepIntervalMs;
}