/// <summary> /// KafkaClientHelperWrapper produces or consumes messages to/from a particular partition of a topic /// </summary> /// <param name="topic">target topic</param> /// <param name="helperConfiguration"></param> public KafkaClientHelperWrapper(string topic, KafkaClientHelperConfiguration helperConfiguration) { if (string.IsNullOrEmpty(topic)) { throw new System.ArgumentNullException("topicParam"); } if (helperConfiguration == null) { throw new System.ArgumentNullException("helperConfiguration"); } if (helperConfiguration.LeaderConfig == null && string.IsNullOrEmpty(helperConfiguration.KafkaBrokerList) && helperConfiguration.ZookeeperConfig == null) { throw new System.ArgumentException("Leader and KafkaBrokerList and Zookeepr connection string are missing"); } Logger.DebugFormat("KafkaClientHelperWrapper constructor start,topicParam={0},kafkaBrokerList={1},zookeeper={2}", topic, helperConfiguration.KafkaBrokerList, helperConfiguration.ZookeeperConfig); ServicePointManager.DefaultConnectionLimit = 5000; ServicePointManager.UseNagleAlgorithm = false; this.topic = topic; this.HelperConfiguration = helperConfiguration; if (this.HelperConfiguration.LeaderConfig != null) { this.leaders.Add(this.HelperConfiguration.LeaderPartition, this.HelperConfiguration.LeaderConfig); this.lastTimeLeaderFound = DateTime.Now; } this.topicMetaRequest = TopicMetadataRequest.Create( new string[] { this.topic }, // topic 0, // API version id this.random.Next(int.MinValue, int.MaxValue), // correlation id Assembly.GetExecutingAssembly().ManifestModule.ToString()); // client id }
public static PullResponse PullMessage( ConsumerConfiguration consumerConfig, BrokerConfiguration leaderBrokerConfig, int correlationID, string topic, int partitionIndex, long fetchOffset, KafkaClientHelperConfiguration helperConfiguration, out long offsetNew) { offsetNew = -1; PullResponse result = null; int payloadCount = 0; // at least retry once int maxRetry = 1; int retryCount = 0; string s = string.Empty; bool success = false; while (!success && retryCount < maxRetry) { try { var requestMap = new Dictionary <string, List <PartitionFetchInfo> >(); requestMap.Add( topic, new List <PartitionFetchInfo>() { new PartitionFetchInfo( partitionIndex, fetchOffset, helperConfiguration.FetchSize) }); using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port)) { var response = consumer.Fetch(new FetchRequest( correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id helperConfiguration.MaxWaitTime, helperConfiguration.MinWaitBytes, requestMap)); if (response == null) { throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } var partitionData = response.PartitionData(topic, partitionIndex); if (partitionData == null) { throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode) { s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return(null); } if (partitionData.Error != ErrorMapping.NoError) { s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return(null); } var messages = partitionData.MessageSet.Messages; s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); success = true; result = new PullResponse(partitionData); if (null != messages && messages.Count() > 0) { payloadCount = messages.Count(); long lastOffset = messages.Last().Offset; if ((payloadCount + fetchOffset) != (lastOffset + 1)) { s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry); } offsetNew = messages.Last().Offset + 1; } return(result); } } catch (Exception) { if (retryCount >= maxRetry) { throw; } } finally { retryCount++; } } // end of while loop return(result); }