public async Task <bool> StartConsuming(CancellationTokenSource cancellationTokenSource)
{
#region Thread to Get File details from DB and Process them
int TimerInterval = int.Parse(System.Configuration.ConfigurationManager.AppSettings["TimerInterval"]);
try
{
while (!cancellationTokenSource.IsCancellationRequested)
{
ProcessKafkaMessage.Process_StgKafkaData();
await Task.Delay(TimerInterval, cancellationTokenSource.Token);
}
}
catch (OperationCanceledException)
{
Console.WriteLine("Canceled!");
}
return(true);
#endregion Thread to Get File details from DB and Process them
}
public async Task <bool> StartPolling(CancellationTokenSource cancellationTokenSource)
{
//Creating local function to recall
int TimerInterval = int.Parse(System.Configuration.ConfigurationManager.AppSettings["TimerInterval"]);
try
{
Log("Start PollingData : " + DateTime.Now.ToString());
PollingData();
Log("End PollingData");
Log("Process_StgKafkaData : Start Process Data : " + DateTime.Now.ToString());
int count = ProcessKafkaMessage.GetPollDataCount("PRODUCTACTIVITY.PUB");
while (count > 0)
{
ProcessKafkaMessage.Process_ActivityStgKafkaData();
count = ProcessKafkaMessage.GetPollDataCount("PRODUCTACTIVITY.PUB");
}
Log("Process_StgKafkaData : END Process Data : " + DateTime.Now.ToString());
if (count == 0)
{
Log("Start RePollingData : Start : " + DateTime.Now.ToString());
await Task.Delay(TimerInterval, cancellationTokenSource.Token);
StartPolling(cancellationTokenSource);
}
}
catch (Exception ex)
{
Log(ex.Message);
Log(ex.StackTrace);
}
void PollingData()
{
var requestObject = new DC_M_masterattribute
{
MasterFor = "Kafka",
PageNo = 0,
PageSize = int.MaxValue
};
try
{
IList <DC_M_masterattribute> Kafka = Proxy.Post <IList <DC_M_masterattribute>, DC_M_masterattribute>(System.Configuration.ConfigurationManager.AppSettings["GetMasterAttributes"], requestObject).GetAwaiter().GetResult();
Log("Get Kafka variable Start");
IList <DC_M_masterattributevalue> KafkaVariables = new List <DC_M_masterattributevalue>();
if (Kafka != null)
{
Guid MasterAttribute_Id = Kafka.Where(w => w.Name == "KafkaVariables").Select(s => s.MasterAttribute_Id).FirstOrDefault();
KafkaVariables = Proxy.Get <IList <DC_M_masterattributevalue> >(string.Format(System.Configuration.ConfigurationManager.AppSettings["GetAllAttributeValuesByMasterId"].ToString(), MasterAttribute_Id.ToString(), int.MaxValue, 0)).GetAwaiter().GetResult();
}
Log("Get Kafka variable End");
var mode = "poll";
Dictionary <string, object> constructConfig = new Dictionary <string, object>();
var topics = new List <string>();
if (KafkaVariables != null)
{
Log("Construct config start");
KafkaVariables = KafkaVariables.Where(w => w.IsActive == "Y").ToList();
constructConfig.Add("group.id", KafkaVariables.Where(w => w.AttributeValue == "acc.group.id").Select(s => s.OTA_CodeTableValue).FirstOrDefault());
constructConfig.Add("enable.auto.commit", KafkaVariables.Where(w => w.AttributeValue == "enable.auto.commit").Select(s => s.OTA_CodeTableValue).FirstOrDefault());
constructConfig.Add("auto.commit.interval.ms", KafkaVariables.Where(w => w.AttributeValue == "auto.commit.interval.ms").Select(s => s.OTA_CodeTableValue).FirstOrDefault());
constructConfig.Add("statistics.interval.ms", KafkaVariables.Where(w => w.AttributeValue == "statistics.interval.ms").Select(s => s.OTA_CodeTableValue).FirstOrDefault());
var bootstrap = KafkaVariables.Where(w => w.AttributeValue.StartsWith("bootstrap.servers")).Select(s => s.OTA_CodeTableValue).ToList();
constructConfig.Add("bootstrap.servers", string.Join(",", bootstrap));
constructConfig.Add("default.topic.config", new Dictionary <string, object>()
{
{ "auto.offset.reset", KafkaVariables.Where(w => w.AttributeValue == "auto.offset.reset").Select(s => s.OTA_CodeTableValue).FirstOrDefault() }
});
topics = KafkaVariables.Where(w => w.AttributeValue.StartsWith("topic.act")).Select(s => s.OTA_CodeTableValue).ToList();
Log("Construct config End");
}
switch (mode)
{
case "poll":
AdvancedConsumer.Run_Poll(constructConfig, topics, cancellationTokenSource);
break;
case "consume":
AdvancedConsumer.Run_Consume(constructConfig, topics, cancellationTokenSource);
break;
default:
AdvancedConsumer.PrintUsage();
break;
}
}
catch (Exception ex)
{
Log("Exception Occurs ");
Log(ex.ToString());
throw;
}
}
return(true);
}
public static void Run_Poll(Dictionary <string, object> constructConfig, List <string> topics, CancellationTokenSource cancellationTokenSource)
{
StartProcess sp = new StartProcess();
try
{
using (var consumer = new Confluent.Kafka.Consumer <Confluent.Kafka.Null, string>(constructConfig, null, new StringDeserializer(Encoding.UTF8)))
{
sp.Log("Run_Poll Start");
// Note: All event handlers are called on the main thread.
consumer.OnMessage += (_, msg) =>
{ /*sp.Log(msg.Value)*/;
ProcessKafkaMessage.InsertInto_StgKafka(msg); };
consumer.OnPartitionEOF += (_, end) =>
{
sp.Log($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
checkReachedEnd = true;
};
// Raised on critical errors, e.g. connection failures or all brokers down.
consumer.OnError += (_, error) =>
{
sp.Log($"Error: {error}");
Console.WriteLine($"Error: {error}");
};
// Raised on deserialization errors or when a consumed message has an error != NoError.
consumer.OnConsumeError += (_, msg) =>
{
sp.Log($"Error consuming from topic/partition/offset {msg.Topic}/{msg.Partition}/{msg.Offset}: {msg.Error}");
Console.WriteLine($"Error consuming from topic/partition/offset {msg.Topic}/{msg.Partition}/{msg.Offset}: {msg.Error}");
};
consumer.OnOffsetsCommitted += (_, commit) =>
{
Console.WriteLine($"[{string.Join(", ", commit.Offsets)}]");
sp.Log(string.Join(", ", commit.Offsets));
if (commit.Error)
{
Console.WriteLine($"Failed to commit offsets: {commit.Error}");
}
Console.WriteLine($"Successfully committed offsets: [{string.Join(", ", commit.Offsets)}]");
};
consumer.OnPartitionsAssigned += (_, partitions) =>
{
Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}");
//partitions.RemoveAt(2);
//partitions.RemoveAt(1);
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (_, partitions) =>
{
Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]");
consumer.Unassign();
};
consumer.OnStatistics += (_, json) => Console.WriteLine($"Statistics: {json}");
consumer.Subscribe(topics);
Console.WriteLine($"Subscribed to: [{string.Join(", ", consumer.Subscription)}]");
//var cancelled = false;
//Console.CancelKeyPress += (_, e) =>
//{
// e.Cancel = true; // prevent the process from terminating.
// cancelled = true;
//};
sp.Log("Berfore Wile loop");
Console.WriteLine("Ctrl-C to exit.");
while (!cancellationTokenSource.IsCancellationRequested)
{
consumer.Poll(TimeSpan.FromMilliseconds(1000));
if (checkReachedEnd)
{
break;
}
}
//consumer.CommitAsync();
sp.Log("Run_Poll End");
}
}
catch (Exception ex)
{
sp.Log("Exeception in Run_Poll ");
sp.Log(ex.ToString());
throw;
}
}
