Constructs a request to send to Kafka to get the current offset for a given topic
Inheritance: AbstractRequest, IWritable
Beispiel #1
0
        public void GetBytesValid()
        {
            string topicName = "topic";
            OffsetRequest request = new OffsetRequest(topicName, 0, OffsetRequest.LatestTime, 10);

            // format = len(request) + requesttype + len(topic) + topic + partition + time + max
            // total byte count = 4 + (2 + 2 + 5 + 4 + 8 + 4)
            MemoryStream ms = new MemoryStream();
            request.WriteTo(ms);
            byte[] bytes = ms.ToArray();
            Assert.IsNotNull(bytes);
            Assert.AreEqual(29, bytes.Length);

            // first 4 bytes = the length of the request
            Assert.AreEqual(25, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0));

            // next 2 bytes = the RequestType which in this case should be Produce
            Assert.AreEqual((short)RequestTypes.Offsets, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0));

            // next 2 bytes = the length of the topic
            Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0));

            // next 5 bytes = the topic
            Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray<byte>()));

            // next 4 bytes = the partition
            Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray<byte>()), 0));

            // next 8 bytes = time
            Assert.AreEqual(OffsetRequest.LatestTime, BitConverter.ToInt64(BitWorks.ReverseBytes(bytes.Skip(17).Take(8).ToArray<byte>()), 0));

            // next 4 bytes = max offsets
            Assert.AreEqual(10, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(25).Take(4).ToArray<byte>()), 0));
        }
Beispiel #2
0
 public static long GetCurrentKafkaOffset(string topic, string address, int port, int partition)
 {
     var request = new OffsetRequest(topic, partition, DateTime.Now.AddDays(-5).Ticks, 10);
     var consumerConfig = new ConsumerConfiguration(address, port);
     IConsumer consumer = new Consumer(consumerConfig, address, port);
     IList<long> list = consumer.GetOffsetsBefore(request);
     return list.Sum();
 }
Beispiel #3
0
 public static long GetCurrentKafkaOffset(string topic, string address, int port)
 {
     OffsetRequest request = new OffsetRequest(topic, 0, DateTime.Now.AddDays(-5).Ticks, 10);
     ConsumerConfig consumerConfig = new ConsumerConfig();
     consumerConfig.Host = address;
     consumerConfig.Port = port;
     IConsumer consumer = new Consumers.Consumer(consumerConfig);
     IList<long> list = consumer.GetOffsetsBefore(request);
     if (list.Count > 0)
     {
         return list[0];
     }
     else
     {
         return 0;
     }
 }
Beispiel #4
0
        private static void GetOffsetsBefore(KafkaConnection conn, OffsetRequest request, IList<long> offsets)
        {
            offsets.Clear(); // to make sure the list is clean after some previous attampts to get data
            conn.Write(request);
            int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0);

            if (dataLength > 0)
            {
                byte[] data = conn.Read(dataLength);

                int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0);
                if (errorCode != KafkaException.NoError)
                {
                    throw new KafkaException(errorCode);
                }

                // skip the error code and process the rest
                byte[] unbufferedData = data.Skip(2).ToArray();

                // first four bytes are the number of offsets
                int numOfOffsets =
                    BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray()), 0);

                for (int ix = 0; ix < numOfOffsets; ix++)
                {
                    int position = (ix * 8) + 4;
                    offsets.Add(
                        BitConverter.ToInt64(
                            BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray()), 0));
                }
            }
        }
Beispiel #5
0
        /// <summary>
        /// Gets a list of valid offsets (up to maxSize) before the given time.
        /// </summary>
        /// <param name="request">
        /// The offset request.
        /// </param>
        /// <returns>
        /// The list of offsets, in descending order.
        /// </returns>
        public IList<long> GetOffsetsBefore(OffsetRequest request)
        {
            var offsets = new List<long>();
            using (var conn = new KafkaConnection(this.Host, this.Port))
            {
                short tryCounter = 1;
                bool success = false;
                while (!success && tryCounter <= this.config.NumberOfTries)
                {
                    try
                    {
                        GetOffsetsBefore(conn, request, offsets);
                        success = true;
                    }
                    catch (Exception ex)
                    {
                        // if maximum number of tries reached
                        if (tryCounter == this.config.NumberOfTries)
                        {
                            throw;
                        }

                        tryCounter++;
                        Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
                    }
                }
            }

            return offsets;
        }
Beispiel #6
0
        private long ResetConsumerOffsets(string topic, Partition partition)
        {
            long offset;
            switch (this.config.AutoOffsetReset)
            {
                case OffsetRequest.SmallestTime:
                    offset = OffsetRequest.EarliestTime;
                    break;
                case OffsetRequest.LargestTime:
                    offset = OffsetRequest.LatestTime;
                    break;
                default:
                    return -1;
            }

            var request = new OffsetRequest(topic, partition.PartId, offset, 1);
            var offsets = this.simpleConsumer.GetOffsetsBefore(request);
            var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic);
            Logger.InfoFormat(CultureInfo.CurrentCulture, "updating partition {0} with {1} offset {2}", partition.Name, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsets[0]);
            ZkUtils.UpdatePersistentPath(this.zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Name, offsets[0].ToString());

            return offsets[0];
        }
Beispiel #7
0
        public void ConsumerGetsOffsets()
        {
            OffsetRequest request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10);

            ConsumerConfig config = new ConsumerConfig(clientConfig);
            IConsumer consumer = new Consumers.Consumer(config);
            IList<long> list = consumer.GetOffsetsBefore(request);

            foreach (long l in list)
            {
                Console.Out.WriteLine(l);
            }
        }
Beispiel #8
0
        /// <summary>
        /// Gets a list of valid offsets (up to maxSize) before the given time.
        /// </summary>
        /// <param name="request">
        /// The offset request.
        /// </param>
        /// <returns>
        /// The list of offsets, in descending order.
        /// </returns>
        public IList<long> GetOffsetsBefore(OffsetRequest request)
        {
            var result = new List<long>();
            short tryCounter = 1;
            while (tryCounter <= this.config.NumberOfTries)
            {
                try
                {
                    using (var conn = new KafkaConnection(
                        this.host,
                        this.port,
                        this.config.BufferSize,
                        this.config.SocketTimeout))
                    {
                        conn.Write(request);
                        int size = conn.Reader.ReadInt32();
                        if (size == 0)
                        {
                            return result;
                        }

                        short errorCode = conn.Reader.ReadInt16();
                        if (errorCode != KafkaException.NoError)
                        {
                            throw new KafkaException(errorCode);
                        }

                        int count = conn.Reader.ReadInt32();
                        for (int i = 0; i < count; i++)
                        {
                            result.Add(conn.Reader.ReadInt64());
                        }

                        return result;
                    }
                }
                catch (Exception ex)
                {
                    //// if maximum number of tries reached
                    if (tryCounter == this.config.NumberOfTries)
                    {
                        throw;
                    }

                    tryCounter++;
                    Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
                }
            }

            return result;
        }
        /// <summary>
        /// Gets a list of valid offsets (up to maxSize) before the given time.
        /// </summary>
        /// <param name="request">
        /// The offset request.
        /// </param>
        /// <returns>
        /// The list of offsets, in descending order.
        /// </returns>
        public OffsetResponse GetOffsetsBefore(OffsetRequest request)
        {
            short tryCounter = 1;
            while (tryCounter <= this.config.NumberOfTries)
            {
                try
                {
                    lock (this)
                    {
                        return connection.Send(request);
                    }
                }
                catch (Exception ex)
                {
                    //// if maximum number of tries reached
                    if (tryCounter == this.config.NumberOfTries)
                    {
                        throw;
                    }

                    tryCounter++;
                    Logger.InfoFormat("GetOffsetsBefore reconnect due to {0}", ex.FormatException());
                }
            }

            return null;
        }
        private static long GetOffset(Consumer consumer, string topic, int partition, long offsetTime)
        {
            string s = string.Empty;
            bool success = false;
            long result = 0;

            if (consumer == null)
            {
                throw new ArgumentNullException("consumer");
            }

            if (string.IsNullOrEmpty(topic))
            {
                throw new ArgumentNullException("topic");
            }

            try
            {
                var offsetRequestInfo = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
                offsetRequestInfo.Add(topic, new List<PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(partition, offsetTime, 128) });
                var offsetRequest = new OffsetRequest(offsetRequestInfo);

                var offsetResponse = consumer.GetOffsetsBefore(offsetRequest);

                if (null == offsetResponse)
                {
                    s = string.Format("OffsetResponse for EarliestTime not found,topic={0}", topic);
                  //  Logger.Error(s);
                    throw new ArgumentException(s);
                }

                List<PartitionOffsetsResponse> partitionOffset = null;
                if (!offsetResponse.ResponseMap.TryGetValue(topic, out partitionOffset)
                    || partitionOffset == null
                    || partitionOffset.Count == 0)
                {
                    s = string.Format("OffsetResponse.ResponseMap for EarliestTime not found,topic={0}", topic);
                  //  Logger.Error(s);
                    throw new ArgumentException(s);
                }

                foreach (var v in partitionOffset)
                {
                    if (v.PartitionId == partition)
                    {
                        result = v.Offsets.First();
                        success = true;
                        break;
                    }
                }

                if (!success)
                {
                    s = string.Format("OffsetResponse.ResponseMap.Partition not found partition={0},topic={1}", partition, topic);
                  //  Logger.Error(s);
                    throw new ArgumentException(s);
                }
            }
            catch (Exception e)
            {
                Logger.Error(string.Format("GetOffset exception,partition={0},topic={1}", partition, topic), e);
                throw;
            }

            return result;
        }