private ProducePerfTestKafkaSimpleManagerWrapper() { config = new KafkaSimpleManagerConfiguration() { Zookeeper = produceOptions.Zookeeper, PartitionerClass = produceOptions.PartitionerClass, MaxMessageSize = SyncProducerConfiguration.DefaultMaxMessageSize }; config.Verify(); producerConfigTemplate = new ProducerConfiguration( new List<BrokerConfiguration>() { }) //The Brokers will be replaced inside of KafkaSimpleManager { ForceToPartition = -1, PartitionerClass = config.PartitionerClass, TotalNumPartitions = 0, RequiredAcks = produceOptions.RequiredAcks, AckTimeout = produceOptions.AckTimeout, SendTimeout = produceOptions.SendTimeout, ReceiveTimeout = produceOptions.ReceiveTimeout, CompressionCodec = KafkaNetLibraryExample.ConvertToCodec(produceOptions.Compression.ToString()), BufferSize = produceOptions.BufferSize, SyncProducerOfOneBroker = produceOptions.SyncProducerOfOneBroker, //Actually it's sync producer socket count of one partition MaxMessageSize = Math.Max(SyncProducerConfiguration.DefaultMaxMessageSize, produceOptions.MessageSize) }; kafkaSimpleManage = new KafkaSimpleManager<byte[], Message>(config); int correlationId = Interlocked.Increment(ref correlationIDGetProducer); kafkaSimpleManage.InitializeProducerPoolForTopic(0, clientId, correlationId, produceOptions.Topic, true, producerConfigTemplate, true); }
public void SetUp() { var brokers = new List<BrokerConfiguration>(); brokers.Add(new BrokerConfiguration { BrokerId = 1, Host = "192.168.0.1", Port = 1234 }); brokers.Add(new BrokerConfiguration { BrokerId = 2, Host = "192.168.0.2", Port = 3456 }); config = new ProducerConfiguration(brokers); }
public ProducerConfiguration(ProducerConfiguration producerConfigTemplate) { this.ForceToPartition = -1; this.SyncProducerOfOneBroker = producerConfigTemplate.SyncProducerOfOneBroker; this.BufferSize = producerConfigTemplate.BufferSize; this.ConnectTimeout = producerConfigTemplate.ConnectTimeout; this.ReceiveTimeout = producerConfigTemplate.ReceiveTimeout; this.SendTimeout = producerConfigTemplate.SendTimeout; this.ReconnectInterval = producerConfigTemplate.ReconnectInterval; this.MaxMessageSize = producerConfigTemplate.MaxMessageSize; this.CompressionCodec = producerConfigTemplate.CompressionCodec; this.CompressedTopics = producerConfigTemplate.CompressedTopics; this.ProducerRetries = producerConfigTemplate.ProducerRetries; this.ProducerRetryExponentialBackoffMinMs = producerConfigTemplate.ProducerRetryExponentialBackoffMinMs; this.ProducerRetryExponentialBackoffMaxMs = producerConfigTemplate.ProducerRetryExponentialBackoffMaxMs; this.ClientId = producerConfigTemplate.ClientId; this.RequiredAcks = producerConfigTemplate.RequiredAcks; this.AckTimeout = producerConfigTemplate.AckTimeout; this.TotalNumPartitions = producerConfigTemplate.TotalNumPartitions; this.TopicMetaDataRefreshIntervalMS = producerConfigTemplate.TopicMetaDataRefreshIntervalMS; this.Verbose = producerConfigTemplate.Verbose; this.partitionerClass = producerConfigTemplate.PartitionerClass; this.Brokers = new List<BrokerConfiguration>(); }
public SyncProducerPool(ProducerConfiguration config) { this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>(); this.Config = config; if (config.ZooKeeper != null) { this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer); this.zkClient.Connect(); } this.AddProducers(config); }
public SyncProducerPool(ProducerConfiguration config, List<ISyncProducer> producers) { this.syncProducers = new ConcurrentDictionary<int, SyncProducerWrapper>(); this.Config = config; if (config.ZooKeeper != null) { this.zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer); this.zkClient.Connect(); } if (producers != null && producers.Any()) { producers.ForEach(x => this.syncProducers.TryAdd(x.Config.BrokerId, new SyncProducerWrapper(x, config.SyncProducerOfOneBroker))); } }
public void ShouldRetryHandleWhenTopicNotFound() { var partitioner = new Mock<IPartitioner<string>>(); var config = new ProducerConfiguration(new List<BrokerConfiguration>()); var pool = new Mock<ISyncProducerPool>(); var producer = new Mock<ISyncProducer>(); var partitionMetadatas = new List<PartitionMetadata>() { new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty<Broker>(), Enumerable.Empty<Broker>()) }; var metadatas = new List<TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.NoError) }; producer.SetupGet(p => p.Config) .Returns( () => new SyncProducerConfiguration(new ProducerConfiguration(new List<BrokerConfiguration>()), 0, "host1", 1234)); producer.Setup(p => p.Send(It.IsAny<TopicMetadataRequest>())).Returns(() => metadatas); var statuses = new Dictionary<TopicAndPartition, ProducerResponseStatus>(); statuses[new TopicAndPartition("test", 0)] = new ProducerResponseStatus(); producer.Setup(p => p.Send(It.IsAny<ProducerRequest>())) .Returns( () => new ProducerResponse(1, statuses)); pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List<ISyncProducer>() { producer.Object }); pool.Setup(p => p.GetProducer(It.IsAny<int>())).Returns(() => producer.Object); var mockPartitionInfo = new Mock<IBrokerPartitionInfo>(); mockPartitionInfo.Setup(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test")) .Returns(() => new List<Partition>()); var handler = new DefaultCallbackHandler<string, Message>(config, partitioner.Object, new DefaultEncoder(), mockPartitionInfo.Object, pool.Object); try { handler.Handle(new List<ProducerData<string, Message>>() { new ProducerData<string, Message>("test", new Message(new byte[100])) }); } catch (FailedToSendMessageException<string>) { mockPartitionInfo.Verify(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test"), Times.Exactly(3)); return; } Assert.Fail("Should have caught exception."); }
public SyncProducerConfiguration(ProducerConfiguration config, int id, string host, int port) { Guard.NotNull(config, "config"); this.Host = host; this.Port = port; this.BrokerId = id; this.BufferSize = config.BufferSize; this.ConnectTimeout = config.ConnectTimeout; this.MaxMessageSize = config.MaxMessageSize; this.ReconnectInterval = config.ReconnectInterval; this.ReceiveTimeout = config.ReceiveTimeout; this.SendTimeout = config.SendTimeout; this.ClientId = config.ClientId; this.CorrelationId = DefaultCorrelationId; this.RequiredAcks = config.RequiredAcks; this.AckTimeout = config.AckTimeout; }
public void ShouldHandleEvents() { var partitioner = new Mock<IPartitioner<string>>(); var config = new ProducerConfiguration(new List<BrokerConfiguration>()); var pool = new Mock<ISyncProducerPool>(); var producer = new Mock<ISyncProducer>(); var partitionMetadatas = new List<PartitionMetadata>() { new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty<Broker>(), Enumerable.Empty<Broker>()) }; var metadatas = new List<TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.NoError) }; producer.SetupGet(p => p.Config) .Returns( () => new SyncProducerConfiguration(new ProducerConfiguration(new List<BrokerConfiguration>()), 0, "host1", 1234)); producer.Setup(p => p.Send(It.IsAny<TopicMetadataRequest>())).Returns(() => metadatas); var statuses = new Dictionary<TopicAndPartition, ProducerResponseStatus>(); statuses[new TopicAndPartition("test", 0)] = new ProducerResponseStatus(); producer.Setup(p => p.Send(It.IsAny<ProducerRequest>())) .Returns( () => new ProducerResponse(1, statuses)); pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List<ISyncProducer>() { producer.Object }); pool.Setup(p => p.GetProducer(It.IsAny<int>())).Returns(() => producer.Object); var mockPartitionInfo = new Mock<IBrokerPartitionInfo>(); mockPartitionInfo.Setup(m => m.GetBrokerPartitionInfo(0, string.Empty, It.IsAny<int>(), "test")) .Returns(() => { var partition = new Partition("test", 0); var replica = new Replica(0, "test"); partition.Leader = replica; return new List<Partition>() { partition }; }); var handler = new DefaultCallbackHandler<string, Message>(config, partitioner.Object, new DefaultEncoder(), mockPartitionInfo.Object, pool.Object); handler.Handle(new List<ProducerData<string, Message>>() { new ProducerData<string, Message>("test", new Message(new byte[100])) }); pool.Verify(p => p.GetProducer(0)); }
/// <summary> /// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class. /// </summary> /// <param name="config">The config.</param> /// <param name="callback">The callback invoked when new broker is added.</param> public ZKBrokerPartitionInfo(ProducerConfiguration config, Action<int, string, int> callback) : this(new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { this.callback = callback; }
/// <summary> /// Initializes a new instance of the <see cref="ConfigBrokerPartitionInfo"/> class. /// </summary> /// <param name="config">The config.</param> public ConfigBrokerPartitionInfo(ProducerConfiguration config) { Guard.NotNull(config, "config"); this.config = config; this.InitializeBrokers(); }
public void AddProducers(ProducerConfiguration config) { var configuredBrokers = config.Brokers.Select(x => new Broker(x.BrokerId, x.Host, x.Port)); if (configuredBrokers.Any()) { configuredBrokers.ForEach(this.AddProducer); } else if (this.zkClient != null) { Logger.DebugFormat("Connecting to {0} for creating sync producers for all brokers in the cluster", config.ZooKeeper.ZkConnect); var brokers = ZkUtils.GetAllBrokersInCluster(this.zkClient); brokers.ForEach(this.AddProducer); } else { throw new IllegalStateException("No producers found from configuration and zk not setup."); } }
/// <summary> /// clear the leader and configurations dependent on the leader /// </summary> public void ClearLeader() { if (this.producer != null) { this.producer.Dispose(); this.producer = null; } if (this.consumer != null) { this.consumer.Dispose(); this.consumer = null; } this.producerConf = null; this.consumerConf = null; this.leaders.Clear(); if (this.PartitionsMetadata != null) { this.PartitionsMetadata = null; } }