static void kafkaConsumer() { OffsetPosition[] offsetPositions = new OffsetPosition[] { new OffsetPosition() { Offset = 1, PartitionId = 0 } }; var options = new KafkaOptions(new Uri("http://localhost:9092"), new Uri("http://localhost:9092")); var consumer = new KafkaNet.Consumer(new ConsumerOptions("test", new BrokerRouter(options)), offsetPositions); using (FileStream fileStream = new FileStream("tweets.txt", FileMode.OpenOrCreate)) { using (StreamWriter writer = new StreamWriter(fileStream)) { using (TextWriter originalConsoleOut = Console.Out) { foreach (var message in consumer.Consume()) { Console.WriteLine("Response: P{0},O{1} : {2}", message.Meta.PartitionId, message.Meta.Offset, Encoding.UTF8.GetString(message.Value)); Console.SetOut(writer); Console.WriteLine(originalConsoleOut); } } } Console.WriteLine("Hello to console only"); } }
/// <summary> /// Rotate the rotor to the next position. /// </summary> /// <returns>true if the next should be rotate</returns> public bool Rotate() { bool shouldAdvance = false; foreach (char item in RotateAt) { if (OffsetPosition.Equals(item)) { shouldAdvance = true; break; } } int offsetIndex = WiresLeft.ProjectCharacter(OffsetPosition); offsetIndex = offsetIndex + 1; if (offsetIndex >= OperatingAlphabet.Count) { offsetIndex = 0; } OffsetPosition = WiresLeft.ProjectIndex(offsetIndex); WiresLeft.Rotate(); WiresRight = WiresLeft.Invert(); return(shouldAdvance); }
private void ReadMessageForever(ConsumerOptions consumerOptions, OffsetPosition[] maxOffsets) { using (var consumer = new Consumer(consumerOptions, maxOffsets)) { var blockingEnumerableOfMessage = consumer.Consume(); foreach (var message in blockingEnumerableOfMessage) { _log.InfoFormat("Offset{0}", message.Meta.Offset); } } }
private static void Consume() { var kafkaRepo = new KafkaConsumerRepository(); bool fromBeginning = Boolean.Parse(ConfigurationManager.AppSettings["FromBeginning"]); var router = InitDefaultConfig(); var consumer = new Consumer(new ConsumerOptions(_topic, router)); //如果我们不想从头开始,使用最新偏移量 if (!fromBeginning) { var maxOffsetByPartition = kafkaRepo.GetOffsetPositionByTopic(_topic); //如果我们得到一个结果使用它,否则默认 if (maxOffsetByPartition.Any()) { var offsets = new List <OffsetPosition>(); foreach (var m in maxOffsetByPartition) { var o = new OffsetPosition(m.Partition, (long)m.MaxOffset + 1); offsets.Add(o); } consumer.SetOffsetPosition(offsets.ToArray()); } else { consumer.SetOffsetPosition(new OffsetPosition()); } } //消耗返回一个阻塞IEnumerable(ie:从没有结束流) foreach (var message in consumer.Consume()) { var messageContent = Encoding.UTF8.GetString(message.Value); Console.WriteLine($"处理带有内容的消息:{messageContent}"); kafkaRepo = new KafkaConsumerRepository(); var consumerMessage = new KafkaConsumerMessage() { Topic = _topic, Offset = (int)message.Meta.Offset, Partition = message.Meta.PartitionId, Content = messageContent, CreatedTime = DateTime.UtcNow }; kafkaRepo.InsertKafkaConsumerMessage(consumerMessage); kafkaRepo.Dispose(); } }
private static void Consume() { KafkaConsumerRepository KafkaRepo = new KafkaConsumerRepository(); bool FromBeginning = Boolean.Parse(ConfigurationManager.AppSettings["FromBeginning"]); var Router = InitDefaultConfig(); var Consumer = new Consumer(new ConsumerOptions(Topic, Router)); //if we don't want to start from beginning, use latest offset. if (!FromBeginning) { var MaxOffsetByPartition = KafkaRepo.GetOffsetPositionByTopic(Topic); //if we get a result use it, otherwise default if (MaxOffsetByPartition.Count != 0) { List <OffsetPosition> offsets = new List <OffsetPosition>(); foreach (var m in MaxOffsetByPartition) { OffsetPosition o = new OffsetPosition(m.Partition, (long)m.MaxOffset + 1); offsets.Add(o); } Consumer.SetOffsetPosition(offsets.ToArray()); } else { Consumer.SetOffsetPosition(new OffsetPosition()); } } //Consume returns a blocking IEnumerable (ie: never ending stream) foreach (var message in Consumer.Consume()) { string MessageContent = Encoding.Default.GetString(message.Value); Console.WriteLine(String.Format("Processing message with content: {0}", MessageContent)); KafkaRepo = new KafkaConsumerRepository(); KafkaConsumerMessage ConsumerMessage = new KafkaConsumerMessage() { Topic = Topic, Offset = (int)message.Meta.Offset, Partition = message.Meta.PartitionId, MessageContent = MessageContent, CreatedAt = DateTime.UtcNow }; KafkaRepo.InsertKafkaConsumerMessage(ConsumerMessage); KafkaRepo.Dispose(); } }
static void ThreadReceive() { string topic2 = "RespostaTopic"; Uri uri2 = new Uri("http://localhost:9092"); var options2 = new KafkaOptions(uri2); var router2 = new BrokerRouter(options2); OffsetPosition[] offsetPositions = new OffsetPosition[] { new OffsetPosition() { Offset = offint, PartitionId = 0 } }; var consumer = new Consumer(new ConsumerOptions(topic2, router2), offsetPositions); string chave = ""; foreach (var message in consumer.Consume()) { if (message.Key != null) { chave = Encoding.ASCII.GetString(message.Key); } if (Receive) { string json = Encoding.ASCII.GetString(message.Value); Console.WriteLine("\n" + json); if (chave != "1") { Receive = false; } } offint = offint + 1; using (StreamWriter file2 = new StreamWriter("Offset.txt", false)) { file2.WriteLine(offint); // Console.WriteLine("Offset:" + offint); file2.Close(); } } }
public OffsetPosition SearchOnGrid(int xMin, int yMin, int xMax, int yMax, int skipChecks, int skipPixels) { int halfWidth = _rimgStart.Width / 2; int halfHeight = _rimgStart.Height / 2; OffsetPosition bestOffsetPosition = null; for (int y = yMin; y < yMax; y += skipChecks) { for (int x = xMin; x < xMax; x += skipChecks) { int offsetX = x - halfWidth; int offsetY = y - halfHeight; double meanSquareError = CalculateMeanSquareError(offsetX, offsetY, skipPixels); if (bestOffsetPosition == null || meanSquareError < bestOffsetPosition.MeanSquareError) { bestOffsetPosition = new OffsetPosition { OffsetX = offsetX, OffsetY = offsetY, MeanSquareError = meanSquareError, }; } } } return(bestOffsetPosition); }
static void Main(string[] args) { var options = new KafkaOptions(new Uri("http://sjkap556:9092"), new Uri("http://sjkap556:9092")); var router = new BrokerRouter(options); OffsetPosition[] offsetPositions = new OffsetPosition[] { new OffsetPosition() { Offset = 0, PartitionId = 0 } }; var consumer = new KafkaNet.Consumer(new ConsumerOptions("testCockpit", new BrokerRouter(options)), offsetPositions); //Consume returns a blocking IEnumerable (ie: never ending stream) foreach (var message in consumer.Consume()) { Console.WriteLine("Response: P{0},O{1} : {2}", message.Meta.PartitionId, message.Meta.Offset, Encoding.UTF8.GetString(message.Value)); } }
public OffsetPosition SearchRecursive(int checkDensity, int skipPixels) { int halfWidth = _rimgStart.Width / 2; int halfHeight = _rimgStart.Height / 2; OffsetPosition bestOffsetPosition = null; int xMin = 0; int yMin = 0; int xMax = _rimgStart.Width; int yMax = _rimgStart.Height; int skipChecks = 1; do { skipChecks = Math.Min((xMax - xMin), (yMax - yMin)) / checkDensity; if (skipChecks <= 0) { skipChecks = 1; } OffsetPosition newOffsetPosition = SearchOnGrid(xMin, yMin, xMax, yMax, skipChecks, skipPixels); if (newOffsetPosition == null) { return(null); } if (bestOffsetPosition == null || newOffsetPosition.MeanSquareError < bestOffsetPosition.MeanSquareError) { bestOffsetPosition = newOffsetPosition; } if (skipChecks > 1) { xMin = (bestOffsetPosition.OffsetX + halfWidth) - skipChecks; yMin = (bestOffsetPosition.OffsetY + halfHeight) - skipChecks; xMax = (bestOffsetPosition.OffsetX + halfWidth) + skipChecks; yMax = (bestOffsetPosition.OffsetY + halfHeight) + skipChecks; } } while (skipChecks > 1); return(bestOffsetPosition); }
/// Thread que pega os comandos do topico do kafka os processa e envia o resultado para o cliente./// static void ThreadProcessaComando() { ////////////////////////////////////////////// string topic = "ComandoTopic4"; Uri uri = new Uri("http://localhost:9092"); var options = new KafkaOptions(uri); var router = new BrokerRouter(options); string topic2 = "RespostaTopic"; Uri uri2 = new Uri("http://localhost:9092"); var options2 = new KafkaOptions(uri); var router2 = new BrokerRouter(options); var produce = new Producer(router2); ////////////////////////////////////////////// //Seta o offset para ler somente mensagens não lidas. OffsetPosition[] offsetPositions = new OffsetPosition[] { new OffsetPosition() { Offset = offint, PartitionId = 0 } }; var consumer = new Consumer(new ConsumerOptions(topic, router), offsetPositions); while (true) { if (!desliga) { //Recebe mensagens(comandos) do tópico foreach (var message in consumer.Consume()) { string json = Encoding.ASCII.GetString(message.Value); Comando comando = JsonConvert.DeserializeObject <Comando>(json); string resposta = ""; if (comando.comand != (int)Comandos.LISTAR) { ProcessaComando(comando, ref resposta); Console.WriteLine("ThreadProcessa: " + resposta); Message msg = new Message(resposta); produce.SendMessageAsync(topic2, new List <Message> { msg }); //Envia resposta do procesamento para cliente. offint = offint + 1; } else if (comando.comand == (int)Comandos.LISTAR) { List <Message> msglist = new List <Message>(); int count = Mapa.Count; int i = 1; foreach (KeyValuePair <long, string> entry in Mapa) { resposta = entry.Key + " - " + entry.Value; if (count == i) { Message msg = new Message(resposta, "2"); msglist.Add(msg); } else { Message msg = new Message(resposta, "1"); msglist.Add(msg); } i++; } produce.SendMessageAsync(topic2, msglist).Wait(); offint = offint + 1; } //Atualiza offset using (StreamWriter file = new StreamWriter("portas.txt", false)) { file.WriteLine(offint); file.Close(); } using (StreamWriter file = new StreamWriter("json.txt", true)) { if (comando.comand == (int)Comandos.READ || comando.comand == (int)Comandos.DESLIGAR || comando.comand == (int)Comandos.LISTAR) { continue; } string comando2 = JsonConvert.SerializeObject(comando); file.WriteLine(comando2); Console.WriteLine("ThreadLog: " + comando2); file.Close(); } } } } }
public static void kafkaConsumer() { OffsetPosition[] offsetPositions = new OffsetPosition[] { new OffsetPosition() { Offset = 1, PartitionId = 0 } }; var options = new KafkaOptions(new Uri("http://*****:*****@'); int indexOfAtSign = tweet.IndexOf("@"); int indexOfColon = tweet.IndexOf(":"); string name = " "; string body = " "; if (indexOfAtSign >= 1 && indexOfColon >= 1) { try { name = tweet.Substring(indexOfAtSign + 1, indexOfColon - indexOfAtSign - 1); } catch (Exception) { name = tweet.Substring(3, 10); } body = tweet.Substring(indexOfColon + 1, tweet.Length - indexOfColon - 1); } var newLine = $"{name},{ dt},{body}"; csv.AppendLine(newLine); File.WriteAllText(@"C:\Users\billm\source\repos\ConsoleApp2\testingTweets.csv", csv.ToString()); } } } Console.WriteLine("Hello to console only"); } }
// Per partition consumer read handler private void ProcessPartitionMessages(object state) { int partition = (int)state; try { Dictionary <uint, MeasurementKey> idTable = new Dictionary <uint, MeasurementKey>(); ConsumerOptions options = new ConsumerOptions(Topic, m_router); LongSynchronizedOperation cacheLastConsumerOffset = null; OffsetPosition consumerCursor = new OffsetPosition { PartitionId = partition, Offset = 0 }; long lastUpdateTime = 0; long lastMetadataUpdateCount = 0; long lastMeasurementTime = 0; options.PartitionWhitelist.Add(partition); options.Log = new TimeSeriesLogger((message, parameters) => OnStatusMessage($"P[{partition}]: " + message, parameters), OnProcessException); // Handle consumer offset tracking, i.e., adapter will start reading messages where it left off from last run if (TrackConsumerOffset) { // Parse path/filename.ext into constituent parts string[] fileParts = new string[3]; fileParts[0] = FilePath.GetDirectoryName(ConsumerOffsetFileName); // 0: path/ fileParts[1] = FilePath.GetFileNameWithoutExtension(ConsumerOffsetFileName); // 1: filename fileParts[2] = FilePath.GetExtension(ConsumerOffsetFileName); // 2: .ext // Include partition index as part of consumer offset cache file name string fileName = $"{fileParts[0]}{fileParts[1]}-P{partition}{fileParts[2]}"; if (File.Exists(fileName)) { try { // Read last consumer offset consumerCursor.Offset = long.Parse(File.ReadAllText(fileName)); } catch (Exception ex) { OnProcessException(new InvalidOperationException($"Failed to read last consumer offset from \"{fileName}\": {ex.Message}", ex)); } } cacheLastConsumerOffset = new LongSynchronizedOperation(() => { // Do not write file any more often than defined consumer offset cache interval int restTime = (int)(Ticks.FromSeconds(ConsumerOffsetCacheInterval) - (DateTime.UtcNow.Ticks - lastUpdateTime)).ToMilliseconds(); if (restTime > 0) { Thread.Sleep(restTime); } lastUpdateTime = DateTime.UtcNow.Ticks; // Write current consumer offset File.WriteAllText(fileName, consumerCursor.Offset.ToString()); }, ex => OnProcessException(new InvalidOperationException($"Failed to cache current consumer offset to \"{fileName}\": {ex.Message}", ex))) { IsBackground = true }; } using (Consumer consumer = new Consumer(options, new OffsetPosition(partition, consumerCursor.Offset))) { lock (m_consumers) m_consumers.Add(new WeakReference <Consumer>(consumer)); foreach (Message message in consumer.Consume()) { if ((object)m_metadata == null) { continue; } uint id; byte metadataVersion; IMeasurement measurement = message.KafkaDeserialize(out id, out metadataVersion); // Kick-off a refresh for new metadata if message version numbers change if (m_lastMetadataVersion != metadataVersion) { m_lastMetadataVersion = metadataVersion; m_updateMetadata.RunOnceAsync(); } // Clear all undefined items in dictionary when metadata gets updated if (lastMetadataUpdateCount < m_metadataUpdateCount) { lastMetadataUpdateCount = m_metadataUpdateCount; foreach (uint undefinedID in idTable.Where(item => item.Value.SignalID == Guid.Empty).Select(item => item.Key).ToArray()) { idTable.Remove(undefinedID); } } // Get associated measurement key, or look it up in metadata table measurement.Key = idTable.GetOrAdd(id, lookupID => MeasurementKey.LookUpBySignalID(m_metadata?.Records?.FirstOrDefault(record => record.ID == lookupID)?.ParseSignalID() ?? Guid.Empty)); // Only publish measurements with associated metadata and are assigned to this adapter if (measurement.Key != MeasurementKey.Undefined && ((object)m_outputMeasurementKeys == null || m_outputMeasurementKeys.Contains(measurement.Key))) { OnNewMeasurements(new[] { measurement }); } // Cache last consumer offset consumerCursor.Offset = message.Offset; if ((object)cacheLastConsumerOffset != null) { cacheLastConsumerOffset.RunOnceAsync(); } if (ReadDelay > -1) { // As a group of measurements transition from timestamp to another, inject configured read delay if (lastMeasurementTime != measurement.Timestamp) { Thread.Sleep(ReadDelay); } lastMeasurementTime = measurement.Timestamp; } } } } catch (Exception ex) { OnProcessException(new InvalidOperationException($"Exception while reading Kafka messages for topic \"{Topic}\" P[{partition}]: {ex.Message}", ex)); } }
// Per partition consumer read handler private void ProcessPartitionMessages(object state) { int partition = (int)state; try { Dictionary<uint, MeasurementKey> idTable = new Dictionary<uint, MeasurementKey>(); ConsumerOptions options = new ConsumerOptions(Topic, m_router); LongSynchronizedOperation cacheLastConsumerOffset = null; OffsetPosition consumerCursor = new OffsetPosition { PartitionId = partition, Offset = 0 }; long lastUpdateTime = 0; long lastMetadataUpdateCount = 0; long lastMeasurementTime = 0; options.PartitionWhitelist.Add(partition); options.Log = new TimeSeriesLogger((message, parameters) => OnStatusMessage(MessageLevel.Info, string.Format($"P[{partition}]: " + message, parameters)), ex => OnProcessException(MessageLevel.Warning, ex)); // Handle consumer offset tracking, i.e., adapter will start reading messages where it left off from last run if (TrackConsumerOffset) { // Parse path/filename.ext into constituent parts string[] fileParts = new string[3]; fileParts[0] = FilePath.GetDirectoryName(ConsumerOffsetFileName); // 0: path/ fileParts[1] = FilePath.GetFileNameWithoutExtension(ConsumerOffsetFileName); // 1: filename fileParts[2] = FilePath.GetExtension(ConsumerOffsetFileName); // 2: .ext // Include partition index as part of consumer offset cache file name string fileName = $"{fileParts[0]}{fileParts[1]}-P{partition}{fileParts[2]}"; if (File.Exists(fileName)) { try { // Read last consumer offset consumerCursor.Offset = long.Parse(File.ReadAllText(fileName)); } catch (Exception ex) { OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed to read last consumer offset from \"{fileName}\": {ex.Message}", ex)); } } cacheLastConsumerOffset = new LongSynchronizedOperation(() => { // Do not write file any more often than defined consumer offset cache interval int restTime = (int)(Ticks.FromSeconds(ConsumerOffsetCacheInterval) - (DateTime.UtcNow.Ticks - lastUpdateTime)).ToMilliseconds(); if (restTime > 0) Thread.Sleep(restTime); lastUpdateTime = DateTime.UtcNow.Ticks; // Write current consumer offset File.WriteAllText(fileName, consumerCursor.Offset.ToString()); }, ex => OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Failed to cache current consumer offset to \"{fileName}\": {ex.Message}", ex))) { IsBackground = true }; } using (Consumer consumer = new Consumer(options, new OffsetPosition(partition, consumerCursor.Offset))) { lock (m_consumers) m_consumers.Add(new WeakReference<Consumer>(consumer)); foreach (Message message in consumer.Consume()) { if ((object)m_metadata == null) continue; uint id; byte metadataVersion; IMeasurement measurement = message.KafkaDeserialize(out id, out metadataVersion); // Kick-off a refresh for new metadata if message version numbers change if (m_lastMetadataVersion != metadataVersion) { m_lastMetadataVersion = metadataVersion; m_updateMetadata.RunOnceAsync(); } // Clear all undefined items in dictionary when metadata gets updated if (lastMetadataUpdateCount < m_metadataUpdateCount) { lastMetadataUpdateCount = m_metadataUpdateCount; foreach (uint undefinedID in idTable.Where(item => item.Value.SignalID == Guid.Empty).Select(item => item.Key).ToArray()) idTable.Remove(undefinedID); } // Get associated measurement key, or look it up in metadata table measurement.Metadata = idTable.GetOrAdd(id, lookupID => MeasurementKey.LookUpBySignalID(m_metadata?.Records?.FirstOrDefault(record => record.ID == lookupID)?.ParseSignalID() ?? Guid.Empty)).Metadata; // Only publish measurements with associated metadata and are assigned to this adapter if (measurement.Key != MeasurementKey.Undefined && ((object)m_outputMeasurementKeys == null || m_outputMeasurementKeys.Contains(measurement.Key))) OnNewMeasurements(new[] { measurement }); // Cache last consumer offset consumerCursor.Offset = message.Offset; if ((object)cacheLastConsumerOffset != null) cacheLastConsumerOffset.RunOnceAsync(); if (ReadDelay > -1) { // As a group of measurements transition from timestamp to another, inject configured read delay if (lastMeasurementTime != measurement.Timestamp) Thread.Sleep(ReadDelay); lastMeasurementTime = measurement.Timestamp; } } } } catch (Exception ex) { OnProcessException(MessageLevel.Warning, new InvalidOperationException($"Exception while reading Kafka messages for topic \"{Topic}\" P[{partition}]: {ex.Message}", ex)); } }
public void Initial() { _task = Task.Run(() => { var options1 = new KafkaOptions(new Uri(_kafkaAddr), new Uri(_kafkaAddr)) { Log = new ConsoleLog() }; var consumer = new Consumer(new ConsumerOptions(_topicName, new BrokerRouter(options1)) { Log = new ConsoleLog() }); // 从数据文件加载 List <KafkaPartOffset> kafkaPartOffset; XmlDataControl.ReadConfig(out kafkaPartOffset); if (kafkaPartOffset != null && kafkaPartOffset.Count > 0) { OffsetPosition[] offsetPositions = new OffsetPosition[kafkaPartOffset.Count]; for (int index = 0; index < kafkaPartOffset.Count; index++) { offsetPositions[index] = new OffsetPosition(kafkaPartOffset[index].PartitionId, kafkaPartOffset[index].Offset); } consumer.SetOffsetPosition(offsetPositions); } foreach (var data in consumer.Consume()) { if (_tokenSource.Token.IsCancellationRequested) { _tokenSource.Token.ThrowIfCancellationRequested(); } else { Log4NetHelper.Instance.Debug("接收Kafka数据成功:" + data.Meta.PartitionId + "-" + data.Meta.Offset + ", data"); // 保存数据到配置文件 XmlDataControl.WriteConfig(data.Meta.PartitionId, data.Meta.Offset); Send2Quere(data.Value); } } }, _tokenSource.Token); _taskSend = Task.Run(() => { var options2 = new KafkaOptions(new Uri(_kafkaAddr)) { Log = new ConsoleLog() }; var producer = new Producer(new BrokerRouter(options2)) { BatchSize = 10, BatchDelayTime = TimeSpan.FromMilliseconds(2000) }; while (true) { try { string message = ResultSpliceQueue.GetFromQueue(); if (string.IsNullOrEmpty(message)) { Thread.Sleep(1000); continue; } else { producer.SendMessageAsync(_outputtopicName, new[] { new Message(message) }).Wait(); Console.WriteLine("Posted messages. AsyncCount:{0}", producer.AsyncCount); Log4NetHelper.Instance.Debug("发送Kafka数据成功:" + message); } } catch (Exception ex) { Log4NetHelper.Instance.Error("从队列发送Kafka数据错误:" + (ex.InnerException != null ? ex.InnerException.Message : ex.Message)); } Thread.Sleep(1000); } }, _tokenSource.Token); }
protected override void EngineController() { logger.LogInformation($"'{this.GetType().Name}' (ID={_EngineID}) Started."); tryRecover: try { var kafkaOptions = new KafkaOptions(new Uri(_Config.URL)); var BrokerRouter = new BrokerRouter(kafkaOptions); var consumerOptions = new ConsumerOptions(Topic, BrokerRouter); //consumerOptions.MaxWaitTimeForMinimumBytes = new TimeSpan(0, 0, 5); //consumerOptions.MinimumBytes = 2; //consumerOptions.FetchBufferMultiplier = 1; //consumerOptions.TopicPartitionQueryTimeMs = 100; int i = 0; var storedOffsetProcessed = GetOffsetProccessed(); if (storedOffsetProcessed == null) { storedOffsetProcessed = new OffsetPosition() { Offset = 0, PartitionId = 0 } } ; using (var consumer = new Consumer(consumerOptions)) { var kafkaOffsets = consumer.GetTopicOffsetAsync(Topic).Result; if (kafkaOffsets != null && kafkaOffsets.Count() != 0) { var kafkaMinOffset = kafkaOffsets.OrderBy(s => s.Offsets.Min()).FirstOrDefault(); var kafkaMaxOffset = kafkaOffsets.OrderByDescending(s => s.Offsets.Max()).FirstOrDefault(); if (storedOffsetProcessed.Offset > kafkaMaxOffset.Offsets.Max() || storedOffsetProcessed.Offset < kafkaMinOffset.Offsets.Min()) { storedOffsetProcessed = new OffsetPosition() { Offset = kafkaMinOffset.Offsets.Min(), PartitionId = kafkaMinOffset.PartitionId } } ; else { storedOffsetProcessed.Offset++; } consumer.SetOffsetPosition(storedOffsetProcessed); } foreach (var message in consumer.Consume(_CancellationToken)) { tryMesseageAgain: try { HandleMessage(message); SaveMesseageOffsetProccessed(message.Meta); } catch (Exception ex) { logger.LogCritical(ex, $"Exception Occured In Kafka Consumer '{this.GetType().Name}' Offset {message.Meta.Offset} (ID={_EngineID})"); var delayTask = Task.Delay(5000); delayTask.Wait(); goto tryMesseageAgain; } } } } catch (OperationCanceledException ex) { } catch (Exception ex) { logger.LogInformation($"Exception Occured In Kafka Consumer '{this.GetType().Name}' (ID={_EngineID}) {ex.Message} \n {ex.StackTrace}."); logger.LogInformation($"'{this.GetType().Name}' (ID={_EngineID}) Recovering..."); goto tryRecover; } logger.LogInformation($"'{this.GetType().Name}' (ID={_EngineID}) Stoped."); }