public KafkaClientHelper(string topicParam, int partitionIndex, KafkaClientHelperConfiguration helperConfiguration) { ServicePointManager.DefaultConnectionLimit = 5000; ServicePointManager.UseNagleAlgorithm = false; this.partitionIndex = partitionIndex; this.helperConfiguration = helperConfiguration; this.MaxMessageSize = helperConfiguration.MaxMessageSize; this.MaxMessagePerSet = helperConfiguration.MaxMessagePerSet; this.ConsumerMaxWait = helperConfiguration.MaxWaitTime; this.Offset = helperConfiguration.Offset; this.OffsetType = helperConfiguration.OffsetType; // Logger.Debug(string.Format("KafkaClientHelper constructor start,topicParam={0},partitionIndex={1},kafkaBrokerList={2}", topicParam, partitionIndex, kafkaBrokerList)); if (string.IsNullOrEmpty(this.helperConfiguration.KafkaBrokerList)) { throw new System.ArgumentNullException("kafkaBrokerList"); } if (string.IsNullOrEmpty(topicParam)) { throw new System.ArgumentNullException("topicParam"); } this.topic = topicParam; // assemble brokerConfig list this.brokerConfList = new List<BrokerConfiguration>(5); string[] brokers = this.helperConfiguration.KafkaBrokerList.Split(new char[] { ',' }); int i = 1; foreach (string v in brokers) { string[] brokerParams = v.Split(new char[] { ':' }); int port = 0; if (brokerParams.Count() == 2 && !string.IsNullOrEmpty(brokerParams[0]) && int.TryParse(brokerParams[1], out port)) { this.brokerConfList.Add(new BrokerConfiguration() { BrokerId = i, Host = brokerParams[0], Port = port }); } i++; } // prepare SyncProducer list i = 0; syncProducerPool = new SyncProducer[this.brokerConfList.Count]; this.RoundRobinSyncProducer(); topicMetaRequest = TopicMetadataRequest.Create( new string[] { this.topic }, // topic 0, // API version id this.random.Next(int.MinValue, int.MaxValue), // correlation id Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id }
public KafkaClientHelper(string topicParam, int partitionIndex, string kafkaBrokerList, KafkaClientHelperConfiguration helperConfiguration) : this(topicParam,partitionIndex,helperConfiguration) { if (!string.IsNullOrEmpty(kafkaBrokerList) && !string.IsNullOrEmpty(helperConfiguration.KafkaBrokerList) && kafkaBrokerList.ToUpperInvariant() != helperConfiguration.KafkaBrokerList.ToUpperInvariant()) { throw new ArgumentException(string.Format("KafkaBrokerList doesn't match, please double check. {0} != {1}", kafkaBrokerList, helperConfiguration.KafkaBrokerList)); } }
public KafkaClientHelper(string topicParam, int partitionIndex, KafkaClientHelperConfiguration helperConfiguration) { ServicePointManager.DefaultConnectionLimit = 5000; ServicePointManager.UseNagleAlgorithm = false; this.partitionIndex = partitionIndex; this.helperConfiguration = helperConfiguration; this.MaxMessageSize = helperConfiguration.MaxMessageSize; this.MaxMessagePerSet = helperConfiguration.MaxMessagePerSet; this.ConsumerMaxWait = helperConfiguration.MaxWaitTime; this.Offset = helperConfiguration.Offset; this.OffsetType = helperConfiguration.OffsetType; // Logger.Debug(string.Format("KafkaClientHelper constructor start,topicParam={0},partitionIndex={1},kafkaBrokerList={2}", topicParam, partitionIndex, kafkaBrokerList)); if (string.IsNullOrEmpty(this.helperConfiguration.KafkaBrokerList)) { throw new System.ArgumentNullException("kafkaBrokerList"); } if (string.IsNullOrEmpty(topicParam)) { throw new System.ArgumentNullException("topicParam"); } this.topic = topicParam; // assemble brokerConfig list this.brokerConfList = new List <BrokerConfiguration>(5); string[] brokers = this.helperConfiguration.KafkaBrokerList.Split(new char[] { ',' }); int i = 1; foreach (string v in brokers) { string[] brokerParams = v.Split(new char[] { ':' }); int port = 0; if (brokerParams.Count() == 2 && !string.IsNullOrEmpty(brokerParams[0]) && int.TryParse(brokerParams[1], out port)) { this.brokerConfList.Add(new BrokerConfiguration() { BrokerId = i, Host = brokerParams[0], Port = port }); } i++; } // prepare SyncProducer list i = 0; syncProducerPool = new SyncProducer[this.brokerConfList.Count]; this.RoundRobinSyncProducer(); topicMetaRequest = TopicMetadataRequest.Create( new string[] { this.topic }, // topic 0, // API version id this.random.Next(int.MinValue, int.MaxValue), // correlation id Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id }
public KafkaClientHelper(string topicParam, int partitionIndex, string kafkaBrokerList, KafkaClientHelperConfiguration helperConfiguration) : this(topicParam, partitionIndex, helperConfiguration) { if (!string.IsNullOrEmpty(kafkaBrokerList) && !string.IsNullOrEmpty(helperConfiguration.KafkaBrokerList) && kafkaBrokerList.ToUpperInvariant() != helperConfiguration.KafkaBrokerList.ToUpperInvariant()) { throw new ArgumentException(string.Format("KafkaBrokerList doesn't match, please double check. {0} != {1}", kafkaBrokerList, helperConfiguration.KafkaBrokerList)); } }
public static PullResponse PullMessage( ConsumerConfiguration consumerConfig, BrokerConfiguration leaderBrokerConfig, int correlationID, string topic, int partitionIndex, long fetchOffset, KafkaClientHelperConfiguration helperConfiguration, out long offsetNew) { offsetNew = -1; PullResponse result = null; int payloadCount = 0; // at least retry once int maxRetry = 1; int retryCount = 0; string s = string.Empty; bool success = false; while (!success && retryCount < maxRetry) { try { var requestMap = new Dictionary<string, List<PartitionFetchInfo>>(); requestMap.Add( topic, new List<PartitionFetchInfo>() { new PartitionFetchInfo( partitionIndex, fetchOffset, helperConfiguration.FetchSize) }); using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port)) { var response = consumer.Fetch(new FetchRequest( correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id helperConfiguration.MaxWaitTime, helperConfiguration.MinWaitBytes, requestMap)); if (response == null) { throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } var partitionData = response.PartitionData(topic, partitionIndex); if (partitionData == null) { throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode) { s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return null; } if (partitionData.Error != ErrorMapping.NoError) { s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return null; } var messages = partitionData.MessageSet.Messages; s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); success = true; result = new PullResponse(partitionData); if (null != messages && messages.Count() > 0) { payloadCount = messages.Count(); long lastOffset = messages.Last().Offset; if ((payloadCount + fetchOffset) != (lastOffset + 1)) { s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry); } offsetNew = messages.Last().Offset + 1; } return result; } } catch (Exception) { if (retryCount >= maxRetry) { throw; } } finally { retryCount++; } } // end of while loop return result; }