/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="topic">The topic to check.</param>
/// <param name="time">time in millisecs (if -1, just get from the latest available)</param>
/// <param name="maxNumOffsets">That maximum number of offsets to return.</param>
/// <param name="correlationId"></param>Id used by the client to identify this transaction. Returned in the response
/// <param name="clientId"></param>Name to identify the client. Used in server logs
/// <param name="partitionId">The partition on the topic.</param>
/// <returns>OffseRersponse</returns>
public OffsetResponse GetOffsetResponse(string topic, long time, int maxNumOffsets, int correlationId, string clientId, int partitionId)
{
var request = new OffsetRequest(correlationId, clientId);
request.AddTopic(topic, partitionId, time, maxNumOffsets);
return(GetOffsetResponseBefore(request));
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="topic">The topic to check.</param>
/// <param name="clientId"></param>Name to identify the client. Used in server logs
/// <param name="partitionId">The partition on the topic.</param>
/// <returns>OffseRersponse containing the first offser for the topic</returns>
public OffsetResponse GetStartOffset(string topic, string clientId, int partitionId)
{
var request = new OffsetRequest(DefaultCorrelationId, clientId);
request.AddTopic(topic, partitionId, OffsetRequest.EarliestTime, 1);
return(GetOffsetResponseBefore(request));
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public IList <long> GetOffsetsBefore(OffsetRequest request)
{
List <long> offsets = new List <long>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
// TODO: need to check in on kafka error codes...assume all's good for now
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray <byte>()), 0);
int position = 0;
for (int ix = 0; ix < numOfOffsets; ix++)
{
position = (ix * 8) + 4;
offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray <byte>()), 0));
}
}
}
return(offsets);
}
/// <summary>
/// Create an OffsetRequest to get the message offset for each requsted topic/partition
/// </summary>
/// <param name="topics"></param>List of topics to get offset for
/// <param name="time"></param>Used to ask for all messages before a certain time (ms). Specify -1 to receive the latest offsets and -2 to receive the earliest available offset. Note that because offsets are pulled in descending order, asking for the earliest offset will always return you a single element.
/// <param name="maxNumOffsets"></param>Max number of offsets to receive. Returns 2 when time = -1 firsrt and current. Returns 1 when time = -2
/// <param name="correlationId"></param>Id used by the client to identify this transaction. Returned in the response
/// <param name="clientId"></param>Name to identify the client. Used in server logs
/// <param name="partitionId"></param>Requested partition
/// <returns></returns>
public OffsetResponse GetOffsetResponse(List <string> topics, long?time, int maxNumOffsets, int correlationId, string clientId, int partitionId)
{
var requestTime = time.GetValueOrDefault(OffsetRequest.LatestTime);
var request = new OffsetRequest(correlationId, clientId);
foreach (var topicName in topics)
{
request.AddTopic(topicName, partitionId, requestTime, maxNumOffsets);
}
return(GetOffsetResponseBefore(request));
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public OffsetResponse GetOffsetResponseBefore(OffsetRequest request)
{
using (var connection = new KafkaConnection(server, port))
{
connection.Write(request.GetRequestBytes().ToArray());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength == 0)
{
return(null);
}
byte[] data = connection.Read(dataLength);
var offsetResponse = new OffsetResponse(data);
return(offsetResponse);
}
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public IList <long> GetOffsetsBefore(OffsetRequest request)
{
List <long> offsets = new List <long>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray <byte>()), 0);
if (errorCode != KafkaException.NoError)
{
throw new KafkaException(errorCode);
}
// skip the error code and process the rest
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray <byte>()), 0);
int position = 0;
for (int ix = 0; ix < numOfOffsets; ix++)
{
position = (ix * 8) + 4;
offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray <byte>()), 0));
}
}
}
return(offsets);
}
/// <summary>
/// Writes a offset request to the server.
/// </summary>
/// <remarks>
/// Write timeout is defaulted to infitite.
/// </remarks>
/// <param name="request">The <see cref="OffsetRequest"/> to send to the server.</param>
public void Write(OffsetRequest request)
{
this.EnsuresNotDisposed();
Guard.NotNull(request, "request");
this.Write(request.RequestBuffer.GetBuffer());
}
/// <summary>
/// Writes a offset request to the server.
/// </summary>
/// <remarks>
/// Write timeout is defaulted to infitite.
/// </remarks>
/// <param name="request">The <see cref="OffsetRequest"/> to send to the server.</param>
public OffsetResponse Send(OffsetRequest request)
{
this.EnsuresNotDisposed();
Guard.NotNull(request, "request");
return(this.Handle(request.RequestBuffer.GetBuffer(), new OffsetResponse.Parser()));
}
/// <summary>
/// Get a list of valid offsets (up to maxSize) before the given time.
/// </summary>
/// <param name="request">The offset request.</param>
/// <returns>List of offsets, in descending order.</returns>
public IList<long> GetOffsetsBefore(OffsetRequest request)
{
List<long> offsets = new List<long>();
using (KafkaConnection connection = new KafkaConnection(Server, Port))
{
connection.Write(request.GetBytes());
int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(connection.Read(4)), 0);
if (dataLength > 0)
{
byte[] data = connection.Read(dataLength);
// TODO: need to check in on kafka error codes...assume all's good for now
byte[] unbufferedData = data.Skip(2).ToArray();
// first four bytes are the number of offsets
int numOfOffsets = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray<byte>()), 0);
int position = 0;
for (int ix = 0; ix < numOfOffsets; ix++)
{
position = (ix * 8) + 4;
offsets.Add(BitConverter.ToInt64(BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray<byte>()), 0));
}
}
}
return offsets;
}
