|
public GetOffsetsBefore ( |
||
| request | /// The offset request. /// | |
| return | IList |
|
public static long GetCurrentKafkaOffset(string topic, string address, int port, int partition)
{
var request = new OffsetRequest(topic, partition, DateTime.Now.AddDays(-5).Ticks, 10);
var consumerConfig = new ConsumerConfiguration(address, port);
IConsumer consumer = new Consumer(consumerConfig, address, port);
IList<long> list = consumer.GetOffsetsBefore(request);
return list.Sum();
}
/// <summary>
/// Retrive first or last offset for a given partition.
/// </summary>
/// <remarks>
/// If <see cref="offsetRequestConstant"/> not equal to <see cref="OffsetRequest.LatestTime"/> or <see cref="OffsetRequest.EarliestTime"/> method returns offset before specified number.
/// </remarks>
/// <param name="consumer">Consumer instance</param>
/// <param name="topic">The topic</param>
/// <param name="partitionId">Partition Id</param>
/// <param name="offsetRequestConstant">Offset that indicates what offset need to return.</param>
/// <returns>
/// Retrive first or last offset for a given partition based on <see cref="offsetRequestConstant"/> parameter.
/// If offset couldn't be retrieved returns null.
/// </returns>
public static long? EarliestOrLatestOffset(Consumer consumer, string topic, int partitionId, long offsetRequestConstant)
{
var requestInfos = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
requestInfos[topic] = new List<PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(partitionId, offsetRequestConstant, 1) };
var offsets = consumer.GetOffsetsBefore(new OffsetRequest(requestInfos));
var topicResult = offsets.ResponseMap[topic];
var responseRow = topicResult != null ? topicResult.FirstOrDefault() : null;
return responseRow != null && responseRow.Offsets != null && responseRow.Offsets.Count > 0 ? responseRow.Offsets[0] : (long?)null;
}
private static long GetOffset(Consumer consumer, string topic, int partition, long offsetTime)
{
string s = string.Empty;
bool success = false;
long result = 0;
if (consumer == null)
{
throw new ArgumentNullException("consumer");
}
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentNullException("topic");
}
try
{
var offsetRequestInfo = new Dictionary <string, List <PartitionOffsetRequestInfo> >();
offsetRequestInfo.Add(topic, new List <PartitionOffsetRequestInfo>()
{
new PartitionOffsetRequestInfo(partition, offsetTime, 128)
});
var offsetRequest = new OffsetRequest(offsetRequestInfo);
var offsetResponse = consumer.GetOffsetsBefore(offsetRequest);
if (null == offsetResponse)
{
s = string.Format("OffsetResponse for EarliestTime not found,topic={0}", topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
List <PartitionOffsetsResponse> partitionOffset = null;
if (!offsetResponse.ResponseMap.TryGetValue(topic, out partitionOffset) ||
partitionOffset == null ||
partitionOffset.Count == 0)
{
s = string.Format("OffsetResponse.ResponseMap for EarliestTime not found,topic={0}", topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
foreach (var v in partitionOffset)
{
if (v.PartitionId == partition)
{
result = v.Offsets.First();
success = true;
break;
}
}
if (!success)
{
s = string.Format("OffsetResponse.ResponseMap.Partition not found partition={0},topic={1}", partition, topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
}
catch (Exception e)
{
Logger.Error(string.Format("GetOffset exception,partition={0},topic={1}", partition, topic), e);
throw;
}
return(result);
}
public void ConsumerGetsOffsets()
{
var consumerConfig = this.ConsumerConfig1;
var request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10);
IConsumer consumer = new Consumer(consumerConfig);
IList<long> list = consumer.GetOffsetsBefore(request);
foreach (long l in list)
{
Console.Out.WriteLine(l);
}
}
private static long GetOffset(Consumer consumer, string topic, int partition, long offsetTime)
{
string s = string.Empty;
bool success = false;
long result = 0;
if (consumer == null)
{
throw new ArgumentNullException("consumer");
}
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentNullException("topic");
}
try
{
var offsetRequestInfo = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
offsetRequestInfo.Add(topic, new List<PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(partition, offsetTime, 128) });
var offsetRequest = new OffsetRequest(offsetRequestInfo);
var offsetResponse = consumer.GetOffsetsBefore(offsetRequest);
if (null == offsetResponse)
{
s = string.Format("OffsetResponse for EarliestTime not found,topic={0}", topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
List<PartitionOffsetsResponse> partitionOffset = null;
if (!offsetResponse.ResponseMap.TryGetValue(topic, out partitionOffset)
|| partitionOffset == null
|| partitionOffset.Count == 0)
{
s = string.Format("OffsetResponse.ResponseMap for EarliestTime not found,topic={0}", topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
foreach (var v in partitionOffset)
{
if (v.PartitionId == partition)
{
result = v.Offsets.First();
success = true;
break;
}
}
if (!success)
{
s = string.Format("OffsetResponse.ResponseMap.Partition not found partition={0},topic={1}", partition, topic);
// Logger.Error(s);
throw new ArgumentException(s);
}
}
catch (Exception e)
{
Logger.Error(string.Format("GetOffset exception,partition={0},topic={1}", partition, topic), e);
throw;
}
return result;
}
