public void ShouldNotUpdateInfoWhenErrorResponse()
{
var pool = new Mock <ISyncProducerPool>();
var producer = new Mock <ISyncProducer>();
var partitionMetadatas = new List <PartitionMetadata>()
{
new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty <Broker>(),
Enumerable.Empty <Broker>())
};
var metadatas = new List <TopicMetadata>()
{
new TopicMetadata("test", partitionMetadatas, ErrorMapping.LeaderNotAvailableCode)
};
producer.Setup(p => p.Send(It.IsAny <TopicMetadataRequest>())).Returns(() => metadatas);
var producerConfig = new SyncProducerConfiguration(new ProducerConfiguration((IList <BrokerConfiguration>)null), 0, "host1", 1234);
producer.Setup(p => p.Config).Returns(producerConfig);
pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List <ISyncProducer>()
{
producer.Object
});
var info = new BrokerPartitionInfo(pool.Object);
info.GetBrokerPartitionInfo(1, "test", 1, "test");
}
public static string SyncProducerConfigToString(SyncProducerConfiguration syncConfig)
{
if (syncConfig == null)
{
return("(SyncProducerConfiguration is null)");
}
return(string.Format("BrokerId={0},Host={1}:Port={2}", syncConfig.BrokerId, syncConfig.Host, syncConfig.Port));
}
public void AddProducer(Broker broker)
{
var syncProducerConfig = new SyncProducerConfiguration(this.Config, broker.Id, broker.Host, broker.Port);
var producerWrapper = new SyncProducerWrapper(syncProducerConfig, this.Config.SyncProducerOfOneBroker);
Logger.DebugFormat("Creating sync producer for broker id = {0} at {1}:{2} SyncProducerOfOneBroker:{3}", broker.Id, broker.Host, broker.Port, this.Config.SyncProducerOfOneBroker);
this.syncProducers.TryAdd(broker.Id, producerWrapper);
}
/// <summary>
/// Initializes a new instance of the <see cref="SyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
public SyncProducer(SyncProducerConfiguration config) : this(config, new KafkaConnection(
config.Host,
config.Port,
config.BufferSize,
config.SendTimeout,
config.ReceiveTimeout,
config.ReconnectInterval))
{
}
public SyncProducerWrapper(SyncProducerConfiguration syncProducerConfig, int count)
{
Producers = new Queue <ISyncProducer>(count);
for (int i = 0; i < count; i++)
{
//TODO: if can't create , should retry later. should not block
Producers.Enqueue(new SyncProducer(syncProducerConfig));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
public SyncProducer(SyncProducerConfiguration config)
: this(config, new KafkaConnection(
config.Host,
config.Port,
config.BufferSize,
config.SendTimeout,
config.ReceiveTimeout,
config.ReconnectInterval))
{
}
/// <summary>
/// Initializes a new instance of the <see cref="SyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
public SyncProducer(SyncProducerConfiguration config)
{
Guard.NotNull(config, "config");
this.Config = config;
this.connection = new KafkaConnection(
this.Config.Host,
this.Config.Port,
config.BufferSize,
config.SocketTimeout);
}
/// <summary>
/// Initializes a new instance of the <see cref="SyncProducer"/> class.
/// </summary>
/// <param name="config">
/// The producer config.
/// </param>
public SyncProducer(SyncProducerConfiguration config)
{
Guard.NotNull(config, "config");
Config = config;
Logger.Debug("Instantiating Sync Producer");
// make time-based reconnect starting at a random time
var randomReconnectInterval = (new Random()).NextDouble() * config.ReconnectInterval;
lastConnectionTime = DateTime.Now - new TimeSpan(Convert.ToInt32(randomReconnectInterval) * TimeSpan.TicksPerMillisecond);
}
/// <summary>
/// Add a new synchronous producer to the pool
/// </summary>
/// <param name="broker">The broker informations.</param>
public override void AddProducer(Broker broker)
{
this.EnsuresNotDisposed();
Guard.NotNull(broker, "broker");
var syncConfig = new SyncProducerConfiguration(this.Config, broker.Id, broker.Host, broker.Port);
var syncProducer = new SyncProducer(syncConfig);
Logger.InfoFormat(
CultureInfo.CurrentCulture,
"Creating sync producer for broker id = {0} at {1}:{2}",
broker.Id,
broker.Host,
broker.Port);
this.syncProducers.Add(broker.Id, syncProducer);
}
public void ShouldThrowMessageTooLarge()
{
var connection = new Mock <IKafkaConnection>();
var config = new SyncProducerConfiguration {
MaxMessageSize = 99
};
var producer = new SyncProducer(config, connection.Object);
producer.Send(new ProducerRequest(1, "client", 0, 0, new List <TopicData>()
{
new TopicData("test",
new List <PartitionData>()
{
new PartitionData(0, new BufferedMessageSet(new List <Message>()
{
new Message(new byte[100])
}, 0))
})
}));
}
public SyncProducer(SyncProducerConfiguration config, IKafkaConnection connection)
{
Guard.NotNull(config, "config");
this.Config = config;
this.connection = connection;
}
/// <summary>
/// Find a leader broker by issuing a topicMetaReqeust
/// </summary>
/// <returns>number of leaders
public int CallKafkaAndFindLeader()
{
string s = string.Empty;
// retry at least once
int maxRetry = 1;
int retryCount = 0;
while (this.leaders.Count == 0 && retryCount < maxRetry)
{
SyncProducerConfiguration producerConfig = null;
try
{
Console.WriteLine("in try catch block");
if (this.syncProducerList == null)
{
// initialize SyncProducerPool
this.InitSyncProducerPool();
}
// retry each broker two times. Observed non-leader broker responds with error message, and some leader knows only a small set of partitions
maxRetry = this.syncProducerList == null ? maxRetry : this.syncProducerList.Count * 2;
if (retryCount > 0)
{
// not the first time, rotate the producer, wait a few seconds, before retry
this.RoundRobinSyncProducer(this.syncProducerList.Count);
Thread.Sleep(this.HelperConfiguration.MaxRetryWaitTime);
}
producerConfig = this.syncProducerList[this.producerIndex].Config;
Console.WriteLine("got producer config");
IEnumerable <TopicMetadata> topicMetaData = this.syncProducerList[this.producerIndex].Send(this.topicMetaRequest);
Console.WriteLine("got topic meta data");
if (topicMetaData.Count() == 0)
{
s = "FindLeader found ZERO count topicMetaData,topic={0},producerIndex[{1}]=[{2}]";
Logger.ErrorFormat(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig));
continue;
}
else if (topicMetaData.First().PartitionsMetadata.Count() == 0)
{
s = "FindLeader found ZERO count partitionMetaData,topic={0},producerIndex[{1}]=[{2}]";
Logger.ErrorFormat(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig));
continue;
}
else if (topicMetaData.Count() > 1)
{
s = "FindLeader found more than one topicData,topicMetaData.Count={0},topic={1},producerIndex[{2}]=[{3}],will use first one - expect only one";
Logger.ErrorFormat(s, topicMetaData.Count(), this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig));
}
this.PartitionsMetadata = topicMetaData.First().PartitionsMetadata.OrderBy(r => r.PartitionId);
this.leaders.Clear();
foreach (var m in this.PartitionsMetadata)
{
this.leaders.Add(m.PartitionId, new BrokerConfiguration()
{
BrokerId = m.Leader.Id,
Host = m.Leader.Host,
Port = m.Leader.Port
});
}
if (this.leaders.Count == 0)
{
s = "no leader found,topic={0},producerIndex[{1}]=[{2}],retryCount={3}";
Logger.ErrorFormat(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig), retryCount);
continue;
}
else
{
s = "KafkaClientHelperWrapper[{0}] leader found,leaders.Count={1},topic={2},producerIndex[{3}]=[{4}],retryCount={5}";
Logger.DebugFormat(s, this.GetHashCode().ToString("X"), this.leaders.Count, this.topic, this.producerIndex,
KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig), retryCount);
this.lastTimeLeaderFound = DateTime.Now;
}
}
catch (Exception e)
{
s = "FindLeader hit exception,topic={0}, producerIndex[{1}]=[{2}],retryCount={3},maxRetry={4}";
Logger.Error(string.Format(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig), retryCount, maxRetry), e);
if (retryCount >= maxRetry)
{
// reach maximum retry, bail
throw;
}
}
finally
{
++retryCount;
}
} // end of while loop
return(this.leaders.Count);
}
public SyncProducer(SyncProducerConfiguration config, IKafkaConnection connection)
{
Guard.NotNull(config, "config");
this.Config = config;
this.connection = connection;
}