public void ShouldNotUpdateInfoWhenErrorResponse() { var pool = new Mock <ISyncProducerPool>(); var producer = new Mock <ISyncProducer>(); var partitionMetadatas = new List <PartitionMetadata>() { new PartitionMetadata(0, new Broker(0, "host1", 1234), Enumerable.Empty <Broker>(), Enumerable.Empty <Broker>()) }; var metadatas = new List <TopicMetadata>() { new TopicMetadata("test", partitionMetadatas, ErrorMapping.LeaderNotAvailableCode) }; producer.Setup(p => p.Send(It.IsAny <TopicMetadataRequest>())).Returns(() => metadatas); var producerConfig = new SyncProducerConfiguration(new ProducerConfiguration((IList <BrokerConfiguration>)null), 0, "host1", 1234); producer.Setup(p => p.Config).Returns(producerConfig); pool.Setup(p => p.GetShuffledProducers()).Returns(() => new List <ISyncProducer>() { producer.Object }); var info = new BrokerPartitionInfo(pool.Object); info.GetBrokerPartitionInfo(1, "test", 1, "test"); }
public static string SyncProducerConfigToString(SyncProducerConfiguration syncConfig) { if (syncConfig == null) { return("(SyncProducerConfiguration is null)"); } return(string.Format("BrokerId={0},Host={1}:Port={2}", syncConfig.BrokerId, syncConfig.Host, syncConfig.Port)); }
public void AddProducer(Broker broker) { var syncProducerConfig = new SyncProducerConfiguration(this.Config, broker.Id, broker.Host, broker.Port); var producerWrapper = new SyncProducerWrapper(syncProducerConfig, this.Config.SyncProducerOfOneBroker); Logger.DebugFormat("Creating sync producer for broker id = {0} at {1}:{2} SyncProducerOfOneBroker:{3}", broker.Id, broker.Host, broker.Port, this.Config.SyncProducerOfOneBroker); this.syncProducers.TryAdd(broker.Id, producerWrapper); }
/// <summary> /// Initializes a new instance of the <see cref="SyncProducer"/> class. /// </summary> /// <param name="config"> /// The producer config. /// </param> public SyncProducer(SyncProducerConfiguration config) : this(config, new KafkaConnection( config.Host, config.Port, config.BufferSize, config.SendTimeout, config.ReceiveTimeout, config.ReconnectInterval)) { }
public SyncProducerWrapper(SyncProducerConfiguration syncProducerConfig, int count) { Producers = new Queue <ISyncProducer>(count); for (int i = 0; i < count; i++) { //TODO: if can't create , should retry later. should not block Producers.Enqueue(new SyncProducer(syncProducerConfig)); } }
/// <summary> /// Initializes a new instance of the <see cref="SyncProducer"/> class. /// </summary> /// <param name="config"> /// The producer config. /// </param> public SyncProducer(SyncProducerConfiguration config) : this(config, new KafkaConnection( config.Host, config.Port, config.BufferSize, config.SendTimeout, config.ReceiveTimeout, config.ReconnectInterval)) { }
/// <summary> /// Initializes a new instance of the <see cref="SyncProducer"/> class. /// </summary> /// <param name="config"> /// The producer config. /// </param> public SyncProducer(SyncProducerConfiguration config) { Guard.NotNull(config, "config"); this.Config = config; this.connection = new KafkaConnection( this.Config.Host, this.Config.Port, config.BufferSize, config.SocketTimeout); }
/// <summary> /// Initializes a new instance of the <see cref="SyncProducer"/> class. /// </summary> /// <param name="config"> /// The producer config. /// </param> public SyncProducer(SyncProducerConfiguration config) { Guard.NotNull(config, "config"); Config = config; Logger.Debug("Instantiating Sync Producer"); // make time-based reconnect starting at a random time var randomReconnectInterval = (new Random()).NextDouble() * config.ReconnectInterval; lastConnectionTime = DateTime.Now - new TimeSpan(Convert.ToInt32(randomReconnectInterval) * TimeSpan.TicksPerMillisecond); }
/// <summary> /// Add a new synchronous producer to the pool /// </summary> /// <param name="broker">The broker informations.</param> public override void AddProducer(Broker broker) { this.EnsuresNotDisposed(); Guard.NotNull(broker, "broker"); var syncConfig = new SyncProducerConfiguration(this.Config, broker.Id, broker.Host, broker.Port); var syncProducer = new SyncProducer(syncConfig); Logger.InfoFormat( CultureInfo.CurrentCulture, "Creating sync producer for broker id = {0} at {1}:{2}", broker.Id, broker.Host, broker.Port); this.syncProducers.Add(broker.Id, syncProducer); }
public void ShouldThrowMessageTooLarge() { var connection = new Mock <IKafkaConnection>(); var config = new SyncProducerConfiguration { MaxMessageSize = 99 }; var producer = new SyncProducer(config, connection.Object); producer.Send(new ProducerRequest(1, "client", 0, 0, new List <TopicData>() { new TopicData("test", new List <PartitionData>() { new PartitionData(0, new BufferedMessageSet(new List <Message>() { new Message(new byte[100]) }, 0)) }) })); }
public SyncProducer(SyncProducerConfiguration config, IKafkaConnection connection) { Guard.NotNull(config, "config"); this.Config = config; this.connection = connection; }
/// <summary> /// Find a leader broker by issuing a topicMetaReqeust /// </summary> /// <returns>number of leaders public int CallKafkaAndFindLeader() { string s = string.Empty; // retry at least once int maxRetry = 1; int retryCount = 0; while (this.leaders.Count == 0 && retryCount < maxRetry) { SyncProducerConfiguration producerConfig = null; try { Console.WriteLine("in try catch block"); if (this.syncProducerList == null) { // initialize SyncProducerPool this.InitSyncProducerPool(); } // retry each broker two times. Observed non-leader broker responds with error message, and some leader knows only a small set of partitions maxRetry = this.syncProducerList == null ? maxRetry : this.syncProducerList.Count * 2; if (retryCount > 0) { // not the first time, rotate the producer, wait a few seconds, before retry this.RoundRobinSyncProducer(this.syncProducerList.Count); Thread.Sleep(this.HelperConfiguration.MaxRetryWaitTime); } producerConfig = this.syncProducerList[this.producerIndex].Config; Console.WriteLine("got producer config"); IEnumerable <TopicMetadata> topicMetaData = this.syncProducerList[this.producerIndex].Send(this.topicMetaRequest); Console.WriteLine("got topic meta data"); if (topicMetaData.Count() == 0) { s = "FindLeader found ZERO count topicMetaData,topic={0},producerIndex[{1}]=[{2}]"; Logger.ErrorFormat(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig)); continue; } else if (topicMetaData.First().PartitionsMetadata.Count() == 0) { s = "FindLeader found ZERO count partitionMetaData,topic={0},producerIndex[{1}]=[{2}]"; Logger.ErrorFormat(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig)); continue; } else if (topicMetaData.Count() > 1) { s = "FindLeader found more than one topicData,topicMetaData.Count={0},topic={1},producerIndex[{2}]=[{3}],will use first one - expect only one"; Logger.ErrorFormat(s, topicMetaData.Count(), this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig)); } this.PartitionsMetadata = topicMetaData.First().PartitionsMetadata.OrderBy(r => r.PartitionId); this.leaders.Clear(); foreach (var m in this.PartitionsMetadata) { this.leaders.Add(m.PartitionId, new BrokerConfiguration() { BrokerId = m.Leader.Id, Host = m.Leader.Host, Port = m.Leader.Port }); } if (this.leaders.Count == 0) { s = "no leader found,topic={0},producerIndex[{1}]=[{2}],retryCount={3}"; Logger.ErrorFormat(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig), retryCount); continue; } else { s = "KafkaClientHelperWrapper[{0}] leader found,leaders.Count={1},topic={2},producerIndex[{3}]=[{4}],retryCount={5}"; Logger.DebugFormat(s, this.GetHashCode().ToString("X"), this.leaders.Count, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig), retryCount); this.lastTimeLeaderFound = DateTime.Now; } } catch (Exception e) { s = "FindLeader hit exception,topic={0}, producerIndex[{1}]=[{2}],retryCount={3},maxRetry={4}"; Logger.Error(string.Format(s, this.topic, this.producerIndex, KafkaClientHelperUtils.SyncProducerConfigToString(producerConfig), retryCount, maxRetry), e); if (retryCount >= maxRetry) { // reach maximum retry, bail throw; } } finally { ++retryCount; } } // end of while loop return(this.leaders.Count); }
public SyncProducer(SyncProducerConfiguration config, IKafkaConnection connection) { Guard.NotNull(config, "config"); this.Config = config; this.connection = connection; }