public void GetBytesValid()
{
string topicName = "topic";
OffsetRequest request = new OffsetRequest(topicName, 0, OffsetRequest.LatestTime, 10);
// format = len(request) + requesttype + len(topic) + topic + partition + time + max
// total byte count = 4 + (2 + 2 + 5 + 4 + 8 + 4)
MemoryStream ms = new MemoryStream();
request.WriteTo(ms);
byte[] bytes = ms.ToArray();
Assert.IsNotNull(bytes);
Assert.AreEqual(29, bytes.Length);
// first 4 bytes = the length of the request
Assert.AreEqual(25, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestTypes.Offsets, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0));
// next 2 bytes = the length of the topic
Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0));
// next 5 bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray<byte>()));
// next 4 bytes = the partition
Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray<byte>()), 0));
// next 8 bytes = time
Assert.AreEqual(OffsetRequest.LatestTime, BitConverter.ToInt64(BitWorks.ReverseBytes(bytes.Skip(17).Take(8).ToArray<byte>()), 0));
// next 4 bytes = max offsets
Assert.AreEqual(10, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(25).Take(4).ToArray<byte>()), 0));
}
public static long GetCurrentKafkaOffset(string topic, string address, int port, int partition)
{
var request = new OffsetRequest(topic, partition, DateTime.Now.AddDays(-5).Ticks, 10);
var consumerConfig = new ConsumerConfiguration(address, port);
IConsumer consumer = new Consumer(consumerConfig, address, port);
IList<long> list = consumer.GetOffsetsBefore(request);
return list.Sum();
}
public static long GetCurrentKafkaOffset(string topic, string address, int port)
{
OffsetRequest request = new OffsetRequest(topic, 0, DateTime.Now.AddDays(-5).Ticks, 10);
ConsumerConfig consumerConfig = new ConsumerConfig();
consumerConfig.Host = address;
consumerConfig.Port = port;
IConsumer consumer = new Consumers.Consumer(consumerConfig);
IList<long> list = consumer.GetOffsetsBefore(request);
if (list.Count > 0)
{
return list[0];
}
else
{
return 0;
}
}
private static void GetOffsetsBefore(KafkaConnection conn, OffsetRequest request, IList<long> offsets)
{
offsets.Clear(); // to make sure the list is clean after some previous attampts to get data
conn.Write(request);
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = conn.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets =
BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray()), 0);
for (int ix = 0; ix < numOfOffsets; ix++)
{
int position = (ix * 8) + 4;
offsets.Add(
BitConverter.ToInt64(
BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray()), 0));
}
}
}
/// <summary>
/// Gets a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">
/// The offset request.
/// </param>
/// <returns>
/// The list of offsets, in descending order.
/// </returns>
public IList<long> GetOffsetsBefore(OffsetRequest request)
{
var offsets = new List<long>();
using (var conn = new KafkaConnection(this.Host, this.Port))
{
short tryCounter = 1;
bool success = false;
while (!success && tryCounter <= this.config.NumberOfTries)
{
try
{
GetOffsetsBefore(conn, request, offsets);
success = true;
}
catch (Exception ex)
{
// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
}
}
}
return offsets;
}
private long ResetConsumerOffsets(string topic, Partition partition)
{
long offset;
switch (this.config.AutoOffsetReset)
{
case OffsetRequest.SmallestTime:
offset = OffsetRequest.EarliestTime;
break;
case OffsetRequest.LargestTime:
offset = OffsetRequest.LatestTime;
break;
default:
return -1;
}
var request = new OffsetRequest(topic, partition.PartId, offset, 1);
var offsets = this.simpleConsumer.GetOffsetsBefore(request);
var topicDirs = new ZKGroupTopicDirs(this.config.GroupId, topic);
Logger.InfoFormat(CultureInfo.CurrentCulture, "updating partition {0} with {1} offset {2}", partition.Name, offset == OffsetRequest.EarliestTime ? "earliest" : "latest", offsets[0]);
ZkUtils.UpdatePersistentPath(this.zkClient, topicDirs.ConsumerOffsetDir + "/" + partition.Name, offsets[0].ToString());
return offsets[0];
}
public void ConsumerGetsOffsets()
{
OffsetRequest request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10);
ConsumerConfig config = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(config);
IList<long> list = consumer.GetOffsetsBefore(request);
foreach (long l in list)
{
Console.Out.WriteLine(l);
}
}
/// <summary>
/// Gets a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">
/// The offset request.
/// </param>
/// <returns>
/// The list of offsets, in descending order.
/// </returns>
public IList<long> GetOffsetsBefore(OffsetRequest request)
{
var result = new List<long>();
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
using (var conn = new KafkaConnection(
this.host,
this.port,
this.config.BufferSize,
this.config.SocketTimeout))
{
conn.Write(request);
int size = conn.Reader.ReadInt32();
if (size == 0)
{
return result;
}
short errorCode = conn.Reader.ReadInt16();
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
int count = conn.Reader.ReadInt32();
for (int i = 0; i < count; i++)
{
result.Add(conn.Reader.ReadInt64());
}
return result;
}
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex);
}
}
return result;
}
/// <summary>
/// Gets a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">
/// The offset request.
/// </param>
/// <returns>
/// The list of offsets, in descending order.
/// </returns>
public OffsetResponse GetOffsetsBefore(OffsetRequest request)
{
short tryCounter = 1;
while (tryCounter <= this.config.NumberOfTries)
{
try
{
lock (this)
{
return connection.Send(request);
}
}
catch (Exception ex)
{
//// if maximum number of tries reached
if (tryCounter == this.config.NumberOfTries)
{
throw;
}
tryCounter++;
Logger.InfoFormat("GetOffsetsBefore reconnect due to {0}", ex.FormatException());
}
}
return null;
}
private static long GetOffset(Consumer consumer, string topic, int partition, long offsetTime)
{
string s = string.Empty;
bool success = false;
long result = 0;
if (consumer == null)
{
throw new ArgumentNullException("consumer");
}
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentNullException("topic");
}
try
{
var offsetRequestInfo = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
offsetRequestInfo.Add(topic, new List<PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(partition, offsetTime, 128) });
var offsetRequest = new OffsetRequest(offsetRequestInfo);
var offsetResponse = consumer.GetOffsetsBefore(offsetRequest);
if (null == offsetResponse)
{
s = string.Format("OffsetResponse for EarliestTime not found,topic={0}", topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
List<PartitionOffsetsResponse> partitionOffset = null;
if (!offsetResponse.ResponseMap.TryGetValue(topic, out partitionOffset)
|| partitionOffset == null
|| partitionOffset.Count == 0)
{
s = string.Format("OffsetResponse.ResponseMap for EarliestTime not found,topic={0}", topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
foreach (var v in partitionOffset)
{
if (v.PartitionId == partition)
{
result = v.Offsets.First();
success = true;
break;
}
}
if (!success)
{
s = string.Format("OffsetResponse.ResponseMap.Partition not found partition={0},topic={1}", partition, topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
}
catch (Exception e)
{
Logger.Error(string.Format("GetOffset exception,partition={0},topic={1}", partition, topic), e);
throw;
}
return result;
}