/// <summary>
/// Reset syncProducer pool
/// </summary>
private void ClearSyncProducerPoolForMetadata()
{
if (this.syncProducerPoolForMetaData != null)
{
this.syncProducerPoolForMetaData.Dispose();
this.syncProducerPoolForMetaData = null;
}
Logger.DebugFormat("KafkaSyncProducerPoolManager[{0}] SyncProducerPool cleared", this.GetHashCode().ToString("X"));
}
public Producer(ProducerConfiguration config)
{
this.Config = config;
syncProducerPool = new SyncProducerPool(config);
this.callbackHandler = new DefaultCallbackHandler <TKey, TData>(config,
ReflectionHelper.Instantiate <IPartitioner <TKey> >(config.PartitionerClass),
ReflectionHelper.Instantiate <IEncoder <TData> >(config.SerializerClass),
new BrokerPartitionInfo(syncProducerPool, topicPartitionInfo, topicPartitionInfoLastUpdateTime, Config.TopicMetaDataRefreshIntervalMS, syncProducerPool.zkClient),
syncProducerPool);
}
/// <summary>
/// Factory method used to instantiating either,
/// synchronous or asynchronous, producer pool based on configuration.
/// </summary>
/// <param name="config">
/// The producer pool configuration.
/// </param>
/// <param name="serializer">
/// The serializer.
/// </param>
/// <returns>
/// Instantiated either, synchronous or asynchronous, producer pool
/// </returns>
public static ProducerPool <TData> CreatePool(ProducerConfiguration config, IEncoder <TData> serializer)
{
if (config.ProducerType == ProducerTypes.Async)
{
return(AsyncProducerPool <TData> .CreateAsyncPool(config, serializer));
}
if (config.ProducerType == ProducerTypes.Sync)
{
return(SyncProducerPool <TData> .CreateSyncPool(config, serializer));
}
throw new InvalidOperationException("Not supported producer type " + config.ProducerType);
}
/// <summary>
/// Initialize SyncProducerPool used for get metadata by query Zookeeper
/// Here only build connection for all ACTIVE broker. So client actually need regularly dispose/recreate or refresh, for example, every 20 minutes..
/// </summary>
public void RecreateSyncProducerPoolForMetadata()
{
SyncProducerPool tempSyncProducerPool = null;
if (this.Config.ZookeeperConfig != null)
{
// Honor Zookeeper connection string only when KafkaBroker list not provided
var producerConfig = new ProducerConfiguration(new List <BrokerConfiguration>());
producerConfig.ZooKeeper = this.Config.ZookeeperConfig;
//This pool only for get metadata, so set SyncProducerOfOneBroker to 1 is enough.
producerConfig.SyncProducerOfOneBroker = 1;
tempSyncProducerPool = new SyncProducerPool(producerConfig);
}
if (this.syncProducerPoolForMetaData != null)
{
lock (syncProducerPoolForMetadataLock)
{
this.syncProducerPoolForMetaData.Dispose();
this.syncProducerPoolForMetaData = null;
this.syncProducerPoolForMetaData = tempSyncProducerPool;
}
}
else
{
this.syncProducerPoolForMetaData = tempSyncProducerPool;
}
if (this.syncProducerPoolForMetaData == null || this.syncProducerPoolForMetaData.Count() == 0)
{
string s = string.Format("KafkaSimpleManager[{0}] SyncProducerPool Initialization produced empty syncProducer list, please check path /brokers/ids in zookeeper={1} to make sure ther is active brokers.",
this.GetHashCode().ToString("X"), this.Config.ZookeeperConfig);
Logger.Error(s);
this.syncProducerPoolForMetaData = null;
throw new ArgumentException(s);
}
else
{
Logger.InfoFormat("The syncProducerPoolForMetaData:{0}", this.syncProducerPoolForMetaData.Count());
foreach (KeyValuePair <int, SyncProducerPool.SyncProducerWrapper> kv in this.syncProducerPoolForMetaData.syncProducers)
{
Logger.InfoFormat("\tBrokerID: {0} count: {1}", kv.Key, kv.Value.Producers.Count);
}
}
Logger.InfoFormat("KafkaSimpleManager[{0}] SyncProducerPool initialized", this.GetHashCode().ToString("X"));
}
/// <summary>
/// Reset syncProducer pool
/// </summary>
private void ClearSyncProducerPool()
{
if (this.syncProducerList != null)
{
this.syncProducerList.Clear();
this.syncProducerList = null;
}
if (this.syncProducerPool != null)
{
this.syncProducerPool.Dispose();
this.syncProducerPool = null;
}
this.producerIndex = 0;
Logger.DebugFormat("KafkaClientHelperWrapper[{0}] SyncProducerPool cleared", this.GetHashCode().ToString("X"));
}
public void Setup()
{
var broker1 = new Mock <ISyncProducer>();
broker1.SetupGet(b => b.Config).Returns(() => new SyncProducerConfiguration()
{
BrokerId = 1
});
var broker2 = new Mock <ISyncProducer>();
broker2.SetupGet(b => b.Config).Returns(() => new SyncProducerConfiguration()
{
BrokerId = 2
});
var brokers = new List <ISyncProducer> {
broker1.Object, broker2.Object
};
pool = new SyncProducerPool(new ProducerConfiguration((List <BrokerConfiguration>)null), brokers);
}
/// <summary>
/// Initialize SyncProducerPool with either the list of brokers provided by the caller or query Zookeeper
/// </summary>
private void InitSyncProducerPool()
{
if (this.syncProducerList != null)
{
// already initialized
return;
}
if (!string.IsNullOrEmpty(this.HelperConfiguration.KafkaBrokerList))
{
// Honor KafkaBrokerList first
this.syncProducerList = new List <ISyncProducer>();
string[] brokers = this.HelperConfiguration.KafkaBrokerList.Split(new char[] { ',' });
int i = 0;
foreach (string v in brokers)
{
i++;
string[] brokerParams = v.Split(new char[] { ':' });
int port = 0;
if (brokerParams.Count() == 2 && !string.IsNullOrEmpty(brokerParams[0]) && int.TryParse(brokerParams[1], out port))
{
var syncProducer = KafkaClientHelperUtils.TryCreateSyncProducer(i, brokerParams[0], port);
if (syncProducer != null)
{
this.syncProducerList.Add(syncProducer);
}
}
}
}
else if (this.HelperConfiguration.ZookeeperConfig != null)
{
// Honor Zookeeper connection string only when KafkaBroker list not provided
var producerConfig = new ProducerConfiguration(new List <BrokerConfiguration>());
producerConfig.ZooKeeper = this.HelperConfiguration.ZookeeperConfig;
//Backward compatible, so set to 1.
producerConfig.SyncProducerOfOneBroker = 1;
this.syncProducerPool = new SyncProducerPool(producerConfig);
this.syncProducerList = this.syncProducerPool.GetShuffledProducers();
}
else if (this.HelperConfiguration.LeaderConfig != null)
{
// if only leader is provided, the leader to be added to syncProducerList
var leader = this.HelperConfiguration.LeaderConfig;
this.syncProducerList = new List <ISyncProducer>()
{
new SyncProducer(new SyncProducerConfiguration()
{
BrokerId = leader.BrokerId, Host = leader.Host, Port = leader.Port
})
};
}
if (this.syncProducerList == null || this.syncProducerList.Count == 0)
{
string s = string.Format("KafkaClientHelperWrapper[{0}] SyncProducerPool Initialization produced empty syncProducer list,Kafka={1},Zookeeper={2}",
this.GetHashCode().ToString("X"), this.HelperConfiguration.KafkaBrokerList, this.HelperConfiguration.ZookeeperConfig);
Logger.Debug(s);
this.syncProducerList = null;
throw new ArgumentException(s);
}
this.RoundRobinSyncProducer(this.syncProducerList.Count);
this.lastTimeBrokerListUpdated = DateTime.Now;
Logger.DebugFormat("KafkaClientHelperWrapper[{0}] SyncProducerPool initialized", this.GetHashCode().ToString("X"));
}
