private ProducePerfTestKafkaSimpleManagerWrapper()
{
config = new KafkaSimpleManagerConfiguration()
{
Zookeeper = produceOptions.Zookeeper,
PartitionerClass = produceOptions.PartitionerClass,
MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize
};
config.Verify();
producerConfigTemplate = new ProducerConfiguration(
new List<BrokerConfiguration>() { }) //The Brokers will be replaced inside of KafkaSimpleManager
{
ForceToPartition = -1,
PartitionerClass = config.PartitionerClass,
TotalNumPartitions = 0,
RequiredAcks = produceOptions.RequiredAcks,
AckTimeout = produceOptions.AckTimeout,
SendTimeout = produceOptions.SendTimeout,
ReceiveTimeout = produceOptions.ReceiveTimeout,
CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()),
BufferSize = produceOptions.BufferSize,
SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition
MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize)
};
kafkaSimpleManage = new KafkaSimpleManager<byte[], Message>(config);
int correlationId = Interlocked.Increment(ref correlationIDGetProducer);
kafkaSimpleManage.InitializeProducerPoolForTopic(0, clientId, correlationId, produceOptions.Topic, true, producerConfigTemplate, true);
}
public void SetUp()
{
var brokers = new List<BrokerConfiguration>();
brokers.Add(new BrokerConfiguration { BrokerId = 1, Host = "192.168.0.1", Port = 1234 });
brokers.Add(new BrokerConfiguration { BrokerId = 2, Host = "192.168.0.2", Port = 3456 });
config = new ProducerConfiguration(brokers);
}
public ProducerConfiguration(ProducerConfiguration producerConfigTemplate)
{
this.ForceToPartition = -1;
this.SyncProducerOfOneBroker = producerConfigTemplate.SyncProducerOfOneBroker;
this.BufferSize = producerConfigTemplate.BufferSize;
this.ConnectTimeout = producerConfigTemplate.ConnectTimeout;
this.ReceiveTimeout = producerConfigTemplate.ReceiveTimeout;
this.SendTimeout = producerConfigTemplate.SendTimeout;
this.ReconnectInterval = producerConfigTemplate.ReconnectInterval;
this.MaxMessageSize = producerConfigTemplate.MaxMessageSize;
this.CompressionCodec = producerConfigTemplate.CompressionCodec;
this.CompressedTopics = producerConfigTemplate.CompressedTopics;
this.ProducerRetries = producerConfigTemplate.ProducerRetries;
this.ProducerRetryExponentialBackoffMinMs = producerConfigTemplate.ProducerRetryExponentialBackoffMinMs;
this.ProducerRetryExponentialBackoffMaxMs = producerConfigTemplate.ProducerRetryExponentialBackoffMaxMs;
this.ClientId = producerConfigTemplate.ClientId;
this.RequiredAcks = producerConfigTemplate.RequiredAcks;
this.AckTimeout = producerConfigTemplate.AckTimeout;
this.TotalNumPartitions = producerConfigTemplate.TotalNumPartitions;
this.TopicMetaDataRefreshIntervalMS = producerConfigTemplate.TopicMetaDataRefreshIntervalMS;
this.Verbose = producerConfigTemplate.Verbose;
this.partitionerClass = producerConfigTemplate.PartitionerClass;
this.Brokers = new List<BrokerConfiguration>();
}
public SyncProducerPool(ProducerConfiguration config)
{
this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>();
this.Config = config;
if (config.ZooKeeper != null)
{
this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer);
this.zkClient.Connect();
}
this.AddProducers(config);
}
public SyncProducerPool(ProducerConfiguration config, List<ISyncProducer> producers)
{
this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>();
this.Config = config;
if (config.ZooKeeper != null)
{
this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer);
this.zkClient.Connect();
}
if (producers != null && producers.Any())
{
producers.ForEach(x => this.syncProducers.TryAdd(x.Config.BrokerId, new SyncProducerWrapper(x, config.SyncProducerOfOneBroker)));
}
}
public void ShouldRetryHandleWhenTopicNotFound()
{
var partitioner = new Mock<IPartitioner<string>>();
var config = new ProducerConfiguration(new List<BrokerConfiguration>());
var pool = new Mock<ISyncProducerPool>();
var producer = new Mock<ISyncProducer>();
var partitionMetadatas = new List<PartitionMetadata>()
{
new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty<Broker>(),
Enumerable.Empty<Broker>())
};
var metadatas = new List<TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.NoError) };
producer.SetupGet(p => p.Config)
.Returns(
() =>
new SyncProducerConfiguration(new ProducerConfiguration(new List<BrokerConfiguration>()), 0,
"host1", 1234));
producer.Setup(p => p.Send(It.IsAny<TopicMetadataRequest>())).Returns(() => metadatas);
var statuses = new Dictionary<TopicAndPartition, ProducerResponseStatus>();
statuses[new TopicAndPartition("test", 0)] = new ProducerResponseStatus();
producer.Setup(p => p.Send(It.IsAny<ProducerRequest>()))
.Returns(
() =>
new ProducerResponse(1, statuses));
pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List<ISyncProducer>() { producer.Object });
pool.Setup(p => p.GetProducer(It.IsAny<int>())).Returns(() => producer.Object);
var mockPartitionInfo = new Mock<IBrokerPartitionInfo>();
mockPartitionInfo.Setup(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test"))
.Returns(() => new List<Partition>());
var handler = new DefaultCallbackHandler<string, Message>(config, partitioner.Object, new DefaultEncoder(), mockPartitionInfo.Object, pool.Object);
try
{
handler.Handle(new List<ProducerData<string, Message>>()
{
new ProducerData<string, Message>("test", new Message(new byte[100]))
});
}
catch (FailedToSendMessageException<string>)
{
mockPartitionInfo.Verify(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test"), Times.Exactly(3));
return;
}
Assert.Fail("Should have caught exception.");
}
public SyncProducerConfiguration(ProducerConfiguration config, int id, string host, int port)
{
Guard.NotNull(config, "config");
this.Host = host;
this.Port = port;
this.BrokerId = id;
this.BufferSize = config.BufferSize;
this.ConnectTimeout = config.ConnectTimeout;
this.MaxMessageSize = config.MaxMessageSize;
this.ReconnectInterval = config.ReconnectInterval;
this.ReceiveTimeout = config.ReceiveTimeout;
this.SendTimeout = config.SendTimeout;
this.ClientId = config.ClientId;
this.CorrelationId = DefaultCorrelationId;
this.RequiredAcks = config.RequiredAcks;
this.AckTimeout = config.AckTimeout;
}
public void ShouldHandleEvents()
{
var partitioner = new Mock<IPartitioner<string>>();
var config = new ProducerConfiguration(new List<BrokerConfiguration>());
var pool = new Mock<ISyncProducerPool>();
var producer = new Mock<ISyncProducer>();
var partitionMetadatas = new List<PartitionMetadata>()
{
new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty<Broker>(),
Enumerable.Empty<Broker>())
};
var metadatas = new List<TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.NoError) };
producer.SetupGet(p => p.Config)
.Returns(
() =>
new SyncProducerConfiguration(new ProducerConfiguration(new List<BrokerConfiguration>()), 0,
"host1", 1234));
producer.Setup(p => p.Send(It.IsAny<TopicMetadataRequest>())).Returns(() => metadatas);
var statuses = new Dictionary<TopicAndPartition, ProducerResponseStatus>();
statuses[new TopicAndPartition("test", 0)] = new ProducerResponseStatus();
producer.Setup(p => p.Send(It.IsAny<ProducerRequest>()))
.Returns(
() =>
new ProducerResponse(1, statuses));
pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List<ISyncProducer>() { producer.Object });
pool.Setup(p => p.GetProducer(It.IsAny<int>())).Returns(() => producer.Object);
var mockPartitionInfo = new Mock<IBrokerPartitionInfo>();
mockPartitionInfo.Setup(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test"))
.Returns(() =>
{
var partition = new Partition("test", 0);
var replica = new Replica(0, "test");
partition.Leader = replica;
return new List<Partition>() { partition };
});
var handler = new DefaultCallbackHandler<string, Message>(config, partitioner.Object, new DefaultEncoder(), mockPartitionInfo.Object, pool.Object);
handler.Handle(new List<ProducerData<string, Message>>() { new ProducerData<string, Message>("test", new Message(new byte[100])) });
pool.Verify(p => p.GetProducer(0));
}
/// <summary>
/// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class.
/// </summary>
/// <param name="config">The config.</param>
/// <param name="callback">The callback invoked when new broker is added.</param>
public ZKBrokerPartitionInfo(ProducerConfiguration config, Action<int, string, int> callback)
: this(new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
{
this.callback = callback;
}
/// <summary>
/// Initializes a new instance of the <see cref="ConfigBrokerPartitionInfo"/> class.
/// </summary>
/// <param name="config">The config.</param>
public ConfigBrokerPartitionInfo(ProducerConfiguration config)
{
Guard.NotNull(config, "config");
this.config = config;
this.InitializeBrokers();
}
public void AddProducers(ProducerConfiguration config)
{
var configuredBrokers = config.Brokers.Select(x => new Broker(x.BrokerId, x.Host, x.Port));
if (configuredBrokers.Any())
{
configuredBrokers.ForEach(this.AddProducer);
}
else if (this.zkClient != null)
{
Logger.DebugFormat("Connecting to {0} for creating sync producers for all brokers in the cluster",
config.ZooKeeper.ZkConnect);
var brokers = ZkUtils.GetAllBrokersInCluster(this.zkClient);
brokers.ForEach(this.AddProducer);
}
else
{
throw new IllegalStateException("No producers found from configuration and zk not setup.");
}
}
/// <summary>
/// clear the leader and configurations dependent on the leader
/// </summary>
public void ClearLeader()
{
if (this.producer != null)
{
this.producer.Dispose();
this.producer = null;
}
if (this.consumer != null)
{
this.consumer.Dispose();
this.consumer = null;
}
this.producerConf = null;
this.consumerConf = null;
this.leaders.Clear();
if (this.PartitionsMetadata != null)
{
this.PartitionsMetadata = null;
}
}