internal static KafkaOffsetType ConvertOffsetType(string offset) { KafkaOffsetType offsetType = KafkaOffsetType.Earliest; if (string.IsNullOrEmpty(offset)) { throw new ArgumentNullException("offset"); } switch (offset.ToLower(CultureInfo.InvariantCulture)) { case "earliest": offsetType = KafkaOffsetType.Earliest; break; case "latest": offsetType = KafkaOffsetType.Latest; break; case "last": offsetType = KafkaOffsetType.Last; break; default: offsetType = KafkaOffsetType.Timestamp; break; } return(offsetType); }
internal static void GetAdjustedOffset <TKey, TData>(string topic, KafkaSimpleManager <TKey, TData> kafkaSimpleManager, int partitionID, KafkaOffsetType offsetType, long offset, int lastMessageCount, out long earliest, out long latest, out long offsetBase) { StringBuilder sbSummaryOnfOnePartition = new StringBuilder(); kafkaSimpleManager.RefreshAndGetOffset(0, string.Empty, 0, topic, partitionID, true, out earliest, out latest); sbSummaryOnfOnePartition.AppendFormat("\t\tearliest:{0}\tlatest:{1}\tlength:{2}" , earliest , latest , (latest - earliest) == 0 ? "(empty)" : (latest - earliest).ToString()); if (offsetType == KafkaOffsetType.Timestamp) { DateTime timestampVal = KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(offset); long timestampLong = KafkaClientHelperUtils.ToUnixTimestampMillis(timestampVal); try { long timeStampOffset = kafkaSimpleManager.RefreshAndGetOffsetByTimeStamp(0, string.Empty, 0, topic, partitionID, timestampVal); sbSummaryOnfOnePartition.AppendFormat("\r\n"); sbSummaryOnfOnePartition.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tTime(Local):{2}\tUnixTimestamp:{3}\t" , timeStampOffset , KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s") , timestampVal.ToString("s") , timestampLong); offsetBase = KafkaClientHelperUtils.GetValidStartReadOffset(offsetType, earliest, latest, timeStampOffset, lastMessageCount); } catch (TimeStampTooSmallException e) { sbSummaryOnfOnePartition.AppendFormat("\r\n"); sbSummaryOnfOnePartition.AppendFormat("\t\ttimeStampOffset:{0}\ttimestamp(UTC):{1}\tTime(Local):{2}\tUnixTimestamp:{3}\t" , "NA since no data before the time you specified, please retry with a bigger value." , KafkaClientHelperUtils.DateTimeFromUnixTimestampMillis(timestampLong).ToString("s") , timestampVal.ToString("s") , timestampLong); throw new ApplicationException(sbSummaryOnfOnePartition.ToString(), e); } } else { offsetBase = KafkaClientHelperUtils.GetValidStartReadOffset(offsetType, earliest, latest, 0, lastMessageCount); } Logger.Info(sbSummaryOnfOnePartition.ToString()); }
public static long GetValidStartReadOffset(KafkaOffsetType offsetType, long earliest, long latest, long offsetTimeStamp, int leastMessageCount) { long offset = 0; ; switch (offsetType) { case KafkaOffsetType.Earliest: offset = earliest; break; case KafkaOffsetType.Last: case KafkaOffsetType.Latest: offset = latest; break; case KafkaOffsetType.Timestamp: offset = offsetTimeStamp; break; default: Logger.ErrorFormat("invalid offsetType={0}", offsetType); throw new ArgumentOutOfRangeException(string.Format("offsetType={0}", offsetType)); } if (offsetType == KafkaOffsetType.Timestamp) { if (offset == latest) { offset -= leastMessageCount; if (offset < earliest) { offset = earliest; } } } else if (offsetType == KafkaOffsetType.Last || offsetType == KafkaOffsetType.Latest) { offset -= leastMessageCount; if (offset < earliest) { offset = earliest; } } return(offset); }
public static long GetValidStartReadOffset(KafkaOffsetType offsetType, long earliest, long latest, long offsetTimeStamp, int leastMessageCount) { long offset = 0; ; switch (offsetType) { case KafkaOffsetType.Earliest: offset = earliest; break; case KafkaOffsetType.Last: case KafkaOffsetType.Latest: offset = latest; break; case KafkaOffsetType.Timestamp: offset = offsetTimeStamp; break; default: Logger.ErrorFormat("invalid offsetType={0}", offsetType); throw new ArgumentOutOfRangeException(string.Format("offsetType={0}", offsetType)); } if (offsetType == KafkaOffsetType.Timestamp) { if (offset == latest) { offset -= leastMessageCount; if (offset < earliest) { offset = earliest; } } } else if (offsetType == KafkaOffsetType.Last || offsetType == KafkaOffsetType.Latest) { offset -= leastMessageCount; if (offset < earliest) { offset = earliest; } } return offset; }
internal static Dictionary <int, string> ConsumeDataOfOnePartitionTotally <TKey, TData>(string topic, KafkaSimpleManager <TKey, TData> kafkaSimpleManager, int partitionID, KafkaOffsetType offsetType, long offsetBase, long earliest, long latest, int lastMessageCount, int wait, long count, string file ) { int totalCount = 0; int correlationID = 0; Dictionary <int, string> dict = new Dictionary <int, string>(); Random rand = new Random(); StringBuilder sb = new StringBuilder(); using (FileStream fs = File.Open(file, FileMode.Append, FileAccess.Write, FileShare.Read)) { using (StreamWriter sw = new StreamWriter(fs)) { #region repeatly consume and dump data long offsetLast = -1; long l = 0; sw.WriteLine("Will read partition {0} from {1}. Earliese:{2} Last:{3} ", partitionID, offsetBase, earliest, latest); Logger.InfoFormat("Will read partition {0} from {1}. Earliese:{2} Last:{3} ", partitionID, offsetBase, earliest, latest); using (Consumer consumer = kafkaSimpleManager.GetConsumer(topic, partitionID)) { while (true) { correlationID++; List <MessageAndOffset> listMessageAndOffsets = new List <MessageAndOffset>(); if (listMessageAndOffsets == null) { Logger.Error("PullMessage got null List<MessageAndOffset>, please check log for detail."); break; } else { offsetLast = listMessageAndOffsets.Last().MessageOffset; #region dump response.Payload foreach (var a in listMessageAndOffsets) { dict.Add((int)a.MessageOffset, Encoding.UTF8.GetString(a.Message.Payload)); } if (listMessageAndOffsets.Any()) { totalCount += listMessageAndOffsets.Count; sw.WriteLine("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest); Logger.InfoFormat("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest); offsetBase = offsetLast + 1; } else { if (offsetBase == latest) { sw.WriteLine("Hit end of queue."); } sw.WriteLine("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest); Logger.InfoFormat("Finish read partition {0} to {1}. Earliese:{2} Last:{3} ", partitionID, offsetLast, earliest, latest); break; } Thread.Sleep(wait); #endregion } l++; if (offsetBase == latest) { break; } } } #endregion Logger.InfoFormat("Topic:{0} Partitoin:{1} Finish Read. Earliest:{2} Latest:{3}, totalCount:{4} " , topic, partitionID, earliest, latest, totalCount); sw.WriteLine("Topic:{0} Partitoin:{1} Finish Read. Earliest:{2} Latest:{3}, totalCount:{4} \r\n " , topic, partitionID, earliest, latest, totalCount); } } return(dict); }