internal static KafkaOffsetType ConvertOffsetType(string offset)
{
KafkaOffsetType offsetType = KafkaOffsetType.Earliest;
if (string.IsNullOrEmpty(offset))
{
throw new ArgumentNullException("offset");
}
switch (offset.ToLower(CultureInfo.InvariantCulture))
{
case "earliest":
offsetType = KafkaOffsetType.Earliest;
break;
case "latest":
offsetType = KafkaOffsetType.Latest;
break;
case "last":
offsetType = KafkaOffsetType.Last;
break;
default:
offsetType = KafkaOffsetType.Timestamp;
break;
}
return(offsetType);
}
internal static void GetAdjustedOffset <TKey, TData>(string topic, KafkaSimpleManager <TKey, TData> kafkaSimpleManager,
int partitionID,
KafkaOffsetType offsetType,
long offset,
int lastMessageCount, out long earliest, out long latest, out long offsetBase)
{
StringBuilder sbSummaryOnfOnePartition = new StringBuilder();
kafkaSimpleManager.RefreshAndGetOffset(0, string.Empty, 0, topic, partitionID, true, out earliest, out latest);
sbSummaryOnfOnePartition.AppendFormat("\t\tearliest:{0}\tlatest:{1}\tlength:{2}"
, earliest
, latest
, (latest - earliest) == 0 ? "(empty)" : (latest - earliest).ToString());
if (offsetType == KafkaOffsetType.Timestamp)
{
DateTime timestampVal = KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(offset);
long timestampLong = KafkaClientHelperUtils.ToUnixTimestampMillis(timestampVal);
try
{
long timeStampOffset = kafkaSimpleManager.RefreshAndGetOffsetByTimeStamp(0, string.Empty, 0, topic, partitionID, timestampVal);
sbSummaryOnfOnePartition.AppendFormat("\r\n");
sbSummaryOnfOnePartition.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tTime(Local):{2}\tUnixTimestamp:{3}\t"
, timeStampOffset
, KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s")
, timestampVal.ToString("s")
, timestampLong);
offsetBase = KafkaClientHelperUtils.GetValidStartReadOffset(offsetType, earliest, latest, timeStampOffset, lastMessageCount);
}
catch (TimeStampTooSmallException e)
{
sbSummaryOnfOnePartition.AppendFormat("\r\n");
sbSummaryOnfOnePartition.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tTime(Local):{2}\tUnixTimestamp:{3}\t"
, "NA since no data before the time you specified, please retry with a bigger value."
, KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s")
, timestampVal.ToString("s")
, timestampLong);
throw new ApplicationException(sbSummaryOnfOnePartition.ToString(), e);
}
}
else
{
offsetBase = KafkaClientHelperUtils.GetValidStartReadOffset(offsetType, earliest, latest, 0, lastMessageCount);
}
Logger.Info(sbSummaryOnfOnePartition.ToString());
}
public static long GetValidStartReadOffset(KafkaOffsetType offsetType,
long earliest,
long latest,
long offsetTimeStamp,
int leastMessageCount)
{
long offset = 0;
;
switch (offsetType)
{
case KafkaOffsetType.Earliest:
offset = earliest;
break;
case KafkaOffsetType.Last:
case KafkaOffsetType.Latest:
offset = latest;
break;
case KafkaOffsetType.Timestamp:
offset = offsetTimeStamp;
break;
default:
Logger.ErrorFormat("invalid offsetType={0}", offsetType);
throw new ArgumentOutOfRangeException(string.Format("offsetType={0}", offsetType));
}
if (offsetType == KafkaOffsetType.Timestamp)
{
if (offset == latest)
{
offset -= leastMessageCount;
if (offset < earliest)
{
offset = earliest;
}
}
}
else if (offsetType == KafkaOffsetType.Last ||
offsetType == KafkaOffsetType.Latest)
{
offset -= leastMessageCount;
if (offset < earliest)
{
offset = earliest;
}
}
return(offset);
}
public static long GetValidStartReadOffset(KafkaOffsetType offsetType, long earliest, long latest, long offsetTimeStamp, int leastMessageCount)
{
long offset = 0; ;
switch (offsetType)
{
case KafkaOffsetType.Earliest:
offset = earliest;
break;
case KafkaOffsetType.Last:
case KafkaOffsetType.Latest:
offset = latest;
break;
case KafkaOffsetType.Timestamp:
offset = offsetTimeStamp;
break;
default:
Logger.ErrorFormat("invalid offsetType={0}", offsetType);
throw new ArgumentOutOfRangeException(string.Format("offsetType={0}", offsetType));
}
if (offsetType == KafkaOffsetType.Timestamp)
{
if (offset == latest)
{
offset -= leastMessageCount;
if (offset < earliest)
{
offset = earliest;
}
}
}
else if (offsetType == KafkaOffsetType.Last
|| offsetType == KafkaOffsetType.Latest)
{
offset -= leastMessageCount;
if (offset < earliest)
{
offset = earliest;
}
}
return offset;
}
internal static Dictionary <int, string> ConsumeDataOfOnePartitionTotally <TKey, TData>(string topic, KafkaSimpleManager <TKey, TData> kafkaSimpleManager,
int partitionID,
KafkaOffsetType offsetType,
long offsetBase,
long earliest,
long latest,
int lastMessageCount,
int wait,
long count,
string file
)
{
int totalCount = 0;
int correlationID = 0;
Dictionary <int, string> dict = new Dictionary <int, string>();
Random rand = new Random();
StringBuilder sb = new StringBuilder();
using (FileStream fs = File.Open(file, FileMode.Append, FileAccess.Write, FileShare.Read))
{
using (StreamWriter sw = new StreamWriter(fs))
{
#region repeatly consume and dump data
long offsetLast = -1;
long l = 0;
sw.WriteLine("Will read partition {0} from {1}. Earliese:{2} Last:{3} ", partitionID, offsetBase, earliest, latest);
Logger.InfoFormat("Will read partition {0} from {1}. Earliese:{2} Last:{3} ", partitionID, offsetBase, earliest, latest);
using (Consumer consumer = kafkaSimpleManager.GetConsumer(topic, partitionID))
{
while (true)
{
correlationID++;
List <MessageAndOffset> listMessageAndOffsets = new List <MessageAndOffset>();
if (listMessageAndOffsets == null)
{
Logger.Error("PullMessage got null List<MessageAndOffset>, please check log for detail.");
break;
}
else
{
offsetLast = listMessageAndOffsets.Last().MessageOffset;
#region dump response.Payload
foreach (var a in listMessageAndOffsets)
{
dict.Add((int)a.MessageOffset, Encoding.UTF8.GetString(a.Message.Payload));
}
if (listMessageAndOffsets.Any())
{
totalCount += listMessageAndOffsets.Count;
sw.WriteLine("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest);
Logger.InfoFormat("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest);
offsetBase = offsetLast + 1;
}
else
{
if (offsetBase == latest)
{
sw.WriteLine("Hit end of queue.");
}
sw.WriteLine("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest);
Logger.InfoFormat("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest);
break;
}
Thread.Sleep(wait);
#endregion
}
l++;
if (offsetBase == latest)
{
break;
}
}
}
#endregion
Logger.InfoFormat("Topic:{0} Partitoin:{1} Finish Read. Earliest:{2} Latest:{3}, totalCount:{4} "
, topic, partitionID, earliest, latest, totalCount);
sw.WriteLine("Topic:{0} Partitoin:{1} Finish Read. Earliest:{2} Latest:{3}, totalCount:{4} \r\n "
, topic, partitionID, earliest, latest, totalCount);
}
}
return(dict);
}
