public async Task <bool> StartConsuming(CancellationTokenSource cancellationTokenSource) { #region Thread to Get File details from DB and Process them int TimerInterval = int.Parse(System.Configuration.ConfigurationManager.AppSettings["TimerInterval"]); try { while (!cancellationTokenSource.IsCancellationRequested) { ProcessKafkaMessage.Process_StgKafkaData(); await Task.Delay(TimerInterval, cancellationTokenSource.Token); } } catch (OperationCanceledException) { Console.WriteLine("Canceled!"); } return(true); #endregion Thread to Get File details from DB and Process them }
public async Task <bool> StartPolling(CancellationTokenSource cancellationTokenSource) { //Creating local function to recall int TimerInterval = int.Parse(System.Configuration.ConfigurationManager.AppSettings["TimerInterval"]); try { Log("Start PollingData : " + DateTime.Now.ToString()); PollingData(); Log("End PollingData"); Log("Process_StgKafkaData : Start Process Data : " + DateTime.Now.ToString()); int count = ProcessKafkaMessage.GetPollDataCount("PRODUCTACTIVITY.PUB"); while (count > 0) { ProcessKafkaMessage.Process_ActivityStgKafkaData(); count = ProcessKafkaMessage.GetPollDataCount("PRODUCTACTIVITY.PUB"); } Log("Process_StgKafkaData : END Process Data : " + DateTime.Now.ToString()); if (count == 0) { Log("Start RePollingData : Start : " + DateTime.Now.ToString()); await Task.Delay(TimerInterval, cancellationTokenSource.Token); StartPolling(cancellationTokenSource); } } catch (Exception ex) { Log(ex.Message); Log(ex.StackTrace); } void PollingData() { var requestObject = new DC_M_masterattribute { MasterFor = "Kafka", PageNo = 0, PageSize = int.MaxValue }; try { IList <DC_M_masterattribute> Kafka = Proxy.Post <IList <DC_M_masterattribute>, DC_M_masterattribute>(System.Configuration.ConfigurationManager.AppSettings["GetMasterAttributes"], requestObject).GetAwaiter().GetResult(); Log("Get Kafka variable Start"); IList <DC_M_masterattributevalue> KafkaVariables = new List <DC_M_masterattributevalue>(); if (Kafka != null) { Guid MasterAttribute_Id = Kafka.Where(w => w.Name == "KafkaVariables").Select(s => s.MasterAttribute_Id).FirstOrDefault(); KafkaVariables = Proxy.Get <IList <DC_M_masterattributevalue> >(string.Format(System.Configuration.ConfigurationManager.AppSettings["GetAllAttributeValuesByMasterId"].ToString(), MasterAttribute_Id.ToString(), int.MaxValue, 0)).GetAwaiter().GetResult(); } Log("Get Kafka variable End"); var mode = "poll"; Dictionary <string, object> constructConfig = new Dictionary <string, object>(); var topics = new List <string>(); if (KafkaVariables != null) { Log("Construct config start"); KafkaVariables = KafkaVariables.Where(w => w.IsActive == "Y").ToList(); constructConfig.Add("group.id", KafkaVariables.Where(w => w.AttributeValue == "acc.group.id").Select(s => s.OTA_CodeTableValue).FirstOrDefault()); constructConfig.Add("enable.auto.commit", KafkaVariables.Where(w => w.AttributeValue == "enable.auto.commit").Select(s => s.OTA_CodeTableValue).FirstOrDefault()); constructConfig.Add("auto.commit.interval.ms", KafkaVariables.Where(w => w.AttributeValue == "auto.commit.interval.ms").Select(s => s.OTA_CodeTableValue).FirstOrDefault()); constructConfig.Add("statistics.interval.ms", KafkaVariables.Where(w => w.AttributeValue == "statistics.interval.ms").Select(s => s.OTA_CodeTableValue).FirstOrDefault()); var bootstrap = KafkaVariables.Where(w => w.AttributeValue.StartsWith("bootstrap.servers")).Select(s => s.OTA_CodeTableValue).ToList(); constructConfig.Add("bootstrap.servers", string.Join(",", bootstrap)); constructConfig.Add("default.topic.config", new Dictionary <string, object>() { { "auto.offset.reset", KafkaVariables.Where(w => w.AttributeValue == "auto.offset.reset").Select(s => s.OTA_CodeTableValue).FirstOrDefault() } }); topics = KafkaVariables.Where(w => w.AttributeValue.StartsWith("topic.act")).Select(s => s.OTA_CodeTableValue).ToList(); Log("Construct config End"); } switch (mode) { case "poll": AdvancedConsumer.Run_Poll(constructConfig, topics, cancellationTokenSource); break; case "consume": AdvancedConsumer.Run_Consume(constructConfig, topics, cancellationTokenSource); break; default: AdvancedConsumer.PrintUsage(); break; } } catch (Exception ex) { Log("Exception Occurs "); Log(ex.ToString()); throw; } } return(true); }
public static void Run_Poll(Dictionary <string, object> constructConfig, List <string> topics, CancellationTokenSource cancellationTokenSource) { StartProcess sp = new StartProcess(); try { using (var consumer = new Confluent.Kafka.Consumer <Confluent.Kafka.Null, string>(constructConfig, null, new StringDeserializer(Encoding.UTF8))) { sp.Log("Run_Poll Start"); // Note: All event handlers are called on the main thread. consumer.OnMessage += (_, msg) => { /*sp.Log(msg.Value)*/; ProcessKafkaMessage.InsertInto_StgKafka(msg); }; consumer.OnPartitionEOF += (_, end) => { sp.Log($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}"); Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}"); checkReachedEnd = true; }; // Raised on critical errors, e.g. connection failures or all brokers down. consumer.OnError += (_, error) => { sp.Log($"Error: {error}"); Console.WriteLine($"Error: {error}"); }; // Raised on deserialization errors or when a consumed message has an error != NoError. consumer.OnConsumeError += (_, msg) => { sp.Log($"Error consuming from topic/partition/offset {msg.Topic}/{msg.Partition}/{msg.Offset}: {msg.Error}"); Console.WriteLine($"Error consuming from topic/partition/offset {msg.Topic}/{msg.Partition}/{msg.Offset}: {msg.Error}"); }; consumer.OnOffsetsCommitted += (_, commit) => { Console.WriteLine($"[{string.Join(", ", commit.Offsets)}]"); sp.Log(string.Join(", ", commit.Offsets)); if (commit.Error) { Console.WriteLine($"Failed to commit offsets: {commit.Error}"); } Console.WriteLine($"Successfully committed offsets: [{string.Join(", ", commit.Offsets)}]"); }; consumer.OnPartitionsAssigned += (_, partitions) => { Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}"); //partitions.RemoveAt(2); //partitions.RemoveAt(1); consumer.Assign(partitions); }; consumer.OnPartitionsRevoked += (_, partitions) => { Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]"); consumer.Unassign(); }; consumer.OnStatistics += (_, json) => Console.WriteLine($"Statistics: {json}"); consumer.Subscribe(topics); Console.WriteLine($"Subscribed to: [{string.Join(", ", consumer.Subscription)}]"); //var cancelled = false; //Console.CancelKeyPress += (_, e) => //{ // e.Cancel = true; // prevent the process from terminating. // cancelled = true; //}; sp.Log("Berfore Wile loop"); Console.WriteLine("Ctrl-C to exit."); while (!cancellationTokenSource.IsCancellationRequested) { consumer.Poll(TimeSpan.FromMilliseconds(1000)); if (checkReachedEnd) { break; } } //consumer.CommitAsync(); sp.Log("Run_Poll End"); } } catch (Exception ex) { sp.Log("Exeception in Run_Poll "); sp.Log(ex.ToString()); throw; } }