public ConsumerConfiguration(string host, int port)
     : this()
 {
     Broker = new BrokerConfiguration {
         Host = host, Port = port
     };
 }
Beispiel #2
0
 private void SetBrokerConfiguration(BrokerConfigurationElement config)
 {
     this.Broker = new BrokerConfiguration
     {
         BrokerId = config.Id,
         Host     = GetIpAddress(config.Host),
         Port     = config.Port
     };
 }
 public static string GetBrokerConfiturationString(int partitionIndex, BrokerConfiguration broker, bool isleader)
 {
     var sb = new StringBuilder();
     if (isleader)
         sb.AppendFormat("partition={0},leader[{0}]={1}", partitionIndex, broker);
     else
         sb.AppendFormat("partition={0},nonleader[{0}]={1}", partitionIndex, broker);
     return sb.ToString();
 }
        public bool Equals(BrokerConfiguration p)
        {
            // If parameter is null return false:
            if ((object)p == null)
            {
                return(false);
            }

            // Return true if the fields match:
            return((BrokerId == p.BrokerId) && (Host == p.Host) && (Port == p.Port));
        }
Beispiel #5
0
        public bool Equals(BrokerConfiguration p)
        {
            // If parameter is null return false:
            if (p == null)
            {
                return(false);
            }

            // Return true if the fields match:
            return(BrokerId == p.BrokerId && Host == p.Host && Port == p.Port);
        }
        public bool Equals(BrokerConfiguration p)
        {
            // If parameter is null return false:
            if ((object)p == null)
            {
                return false;
            }

            // Return true if the fields match:
            return (BrokerId == p.BrokerId) && (Host == p.Host) && (Port == p.Port);
        }
        public static string GetBrokerConfiturationString(int partitionIndex, BrokerConfiguration broker, bool isleader)
        {
            var sb = new StringBuilder();

            if (isleader)
            {
                sb.AppendFormat("partition={0},leader[{0}]={1}", partitionIndex, broker);
            }
            else
            {
                sb.AppendFormat("partition={0},nonleader[{0}]={1}", partitionIndex, broker);
            }
            return(sb.ToString());
        }
        public override bool Equals(System.Object obj)
        {
            // If parameter is null return false.
            if (obj == null)
            {
                return(false);
            }

            // If parameter cannot be cast to Point return false.
            BrokerConfiguration p = obj as BrokerConfiguration;

            if ((System.Object)p == null)
            {
                return(false);
            }

            // Return true if the fields match:
            return((BrokerId == p.BrokerId) && (Host == p.Host) && (Port == p.Port));
        }
        public ConsumerConfiguration(ConsumerConfiguration cosumerConfigTemplate, BrokerConfiguration brokerConfiguration)
        {
            this.Broker = brokerConfiguration;

            this.NumberOfTries        = cosumerConfigTemplate.NumberOfTries;
            this.Timeout              = cosumerConfigTemplate.Timeout;
            this.AutoOffsetReset      = cosumerConfigTemplate.AutoOffsetReset;
            this.AutoCommit           = cosumerConfigTemplate.AutoCommit;
            this.AutoCommitInterval   = cosumerConfigTemplate.AutoCommitInterval;
            this.FetchSize            = cosumerConfigTemplate.FetchSize;
            this.MaxFetchFactor       = cosumerConfigTemplate.MaxFetchFactor;
            this.BackOffIncrement     = cosumerConfigTemplate.BackOffIncrement;
            this.ConsumerId           = cosumerConfigTemplate.ConsumerId;
            this.ReconnectInterval    = cosumerConfigTemplate.ReconnectInterval;
            this.ShutdownTimeout      = cosumerConfigTemplate.ShutdownTimeout;
            this.MaxFetchBufferLength = cosumerConfigTemplate.MaxFetchBufferLength;
            this.SendTimeout          = cosumerConfigTemplate.SendTimeout;
            this.ReceiveTimeout       = cosumerConfigTemplate.ReceiveTimeout;
            this.BufferSize           = cosumerConfigTemplate.BufferSize;
            this.Verbose              = cosumerConfigTemplate.Verbose;
            this.ConsumeGroupRebalanceRetryIntervalMs     = cosumerConfigTemplate.ConsumeGroupRebalanceRetryIntervalMs;
            this.ConsumeGroupFindNewLeaderSleepIntervalMs = cosumerConfigTemplate.ConsumeGroupFindNewLeaderSleepIntervalMs;
        }
        /// <summary>
        /// Convert a broker string into a BrokerConfiguration.
        /// </summary>
        /// <param name="brokerAddr">Broker string must follow the format of {Broker-ID},{Host}:{Port}</param>
        /// <returns></returns>
        public static BrokerConfiguration ToBrokerConfig(string brokerAddr)
        {
            if (string.IsNullOrEmpty(brokerAddr))
            {
                return null;
            }

            string[] brokerParams = brokerAddr.Split(',');
            if (brokerParams == null || brokerParams.Length != 2)
            {
                return null;
            }

            int brokerId = -1;
            if (!int.TryParse(brokerParams[0], out brokerId))
            {
                return null;
            }

            string[] hostParams = brokerParams[1].Split(':');
            if (hostParams == null || hostParams.Length != 2)
            {
                return null;
            }

            int portNumber = -1;
            if (!int.TryParse(hostParams[1], out portNumber))
            {
                return null;
            }

            var broker = new BrokerConfiguration() { BrokerId = brokerId, Host = hostParams[0], Port = portNumber };
            return broker;
        }
        public static PullResponse PullMessage(
            ConsumerConfiguration consumerConfig,
            BrokerConfiguration leaderBrokerConfig,
            int correlationID,
            string topic,
            int partitionIndex,
            long fetchOffset,
            KafkaClientHelperConfiguration helperConfiguration,
            out long offsetNew)
        {
            offsetNew = -1;
            PullResponse result = null;
            int payloadCount = 0;

            // at least retry once
            int maxRetry = 1;
            int retryCount = 0;
            string s = string.Empty;
            bool success = false;
            while (!success && retryCount < maxRetry)
            {
                try
                {
                    var requestMap = new Dictionary<string, List<PartitionFetchInfo>>();
                    requestMap.Add(
                        topic,
                        new List<PartitionFetchInfo>()
                        {
                            new PartitionFetchInfo(
                                partitionIndex,
                                fetchOffset,
                                helperConfiguration.FetchSize)
                        });
                    using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port))
                    {
                        var response = consumer.Fetch(new FetchRequest(
                            correlationID, //random.Next(int.MinValue, int.MaxValue),                        // correlation id
                            Assembly.GetExecutingAssembly().ManifestModule.ToString(),      // client id
                            helperConfiguration.MaxWaitTime,
                            helperConfiguration.MinWaitBytes,
                            requestMap));

                        if (response == null)
                        {
                            throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}",
                                fetchOffset, leaderBrokerConfig, topic, partitionIndex));
                        }

                        var partitionData = response.PartitionData(topic, partitionIndex);
                        if (partitionData == null)
                        {
                            throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}",
                                fetchOffset, leaderBrokerConfig, topic, partitionIndex));
                        }

                        if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
                        {
                            s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}";
                            Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
                            return null;
                        }

                        if (partitionData.Error != ErrorMapping.NoError)
                        {
                            s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
                            Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
                            return null;
                        }

                        var messages = partitionData.MessageSet.Messages;

                        s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
                        Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);

                        success = true;
                        result = new PullResponse(partitionData);

                        if (null != messages && messages.Count() > 0)
                        {
                            payloadCount = messages.Count();
                            long lastOffset = messages.Last().Offset;

                            if ((payloadCount + fetchOffset) != (lastOffset + 1))
                            {
                                s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}";
                                Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry);
                            }
                            offsetNew = messages.Last().Offset + 1;
                        }

                        return result;
                    }
                }
                catch (Exception)
                {
                    if (retryCount >= maxRetry)
                    {
                        throw;
                    }
                }
                finally
                {
                    retryCount++;
                }
            } // end of while loop

            return result;
        }
        public ConsumerConfiguration(ConsumerConfiguration cosumerConfigTemplate, BrokerConfiguration brokerConfiguration)
        {
            this.Broker = brokerConfiguration;

            this.NumberOfTries = cosumerConfigTemplate.NumberOfTries;
            this.Timeout = cosumerConfigTemplate.Timeout;
            this.AutoOffsetReset = cosumerConfigTemplate.AutoOffsetReset;
            this.AutoCommit = cosumerConfigTemplate.AutoCommit;
            this.AutoCommitInterval = cosumerConfigTemplate.AutoCommitInterval;
            this.FetchSize = cosumerConfigTemplate.FetchSize;
            this.MaxFetchFactor = cosumerConfigTemplate.MaxFetchFactor;
            this.BackOffIncrement = cosumerConfigTemplate.BackOffIncrement;
            this.ConsumerId = cosumerConfigTemplate.ConsumerId;
            this.ReconnectInterval = cosumerConfigTemplate.ReconnectInterval;
            this.ShutdownTimeout = cosumerConfigTemplate.ShutdownTimeout;
            this.MaxFetchBufferLength = cosumerConfigTemplate.MaxFetchBufferLength;
            this.SendTimeout = cosumerConfigTemplate.SendTimeout;
            this.ReceiveTimeout = cosumerConfigTemplate.ReceiveTimeout;
            this.BufferSize = cosumerConfigTemplate.BufferSize;
            this.Verbose = cosumerConfigTemplate.Verbose;
            this.ConsumeGroupRebalanceRetryIntervalMs = cosumerConfigTemplate.ConsumeGroupRebalanceRetryIntervalMs;
            this.ConsumeGroupFindNewLeaderSleepIntervalMs = cosumerConfigTemplate.ConsumeGroupFindNewLeaderSleepIntervalMs;
        }