public void InsertKafkaConsumerMessage(KafkaConsumerMessage Message)
{
Console.WriteLine(String.Format("Insert {0}: {1}", Message.KafkaConsumerMessageID.ToString(), Message.MessageContent));
context.KafkaConsumerMessage.Add(Message);
context.SaveChanges();
Console.WriteLine(String.Format("Saved {0}: {1}", Message.KafkaConsumerMessageID.ToString(), Message.MessageContent));
}
private static void Consume()
{
KafkaConsumerRepository KafkaRepo = new KafkaConsumerRepository();
bool FromBeginning = Boolean.Parse(ConfigurationManager.AppSettings["FromBeginning"]);
var Router = InitDefaultConfig();
var Consumer = new Consumer(new ConsumerOptions(Topic, Router));
//if we don't want to start from beginning, use latest offset.
if (!FromBeginning)
{
var MaxOffsetByPartition = KafkaRepo.GetOffsetPositionByTopic(Topic);
//if we get a result use it, otherwise default
if (MaxOffsetByPartition.Count != 0)
{
List <OffsetPosition> offsets = new List <OffsetPosition>();
foreach (var m in MaxOffsetByPartition)
{
OffsetPosition o = new OffsetPosition(m.Partition, (long)m.MaxOffset + 1);
offsets.Add(o);
}
Consumer.SetOffsetPosition(offsets.ToArray());
}
else
{
Consumer.SetOffsetPosition(new OffsetPosition());
}
}
//Consume returns a blocking IEnumerable (ie: never ending stream)
foreach (var message in Consumer.Consume())
{
string MessageContent = Encoding.Default.GetString(message.Value);
Console.WriteLine(String.Format("Processing message with content: {0}", MessageContent));
KafkaRepo = new KafkaConsumerRepository();
KafkaConsumerMessage ConsumerMessage = new KafkaConsumerMessage()
{
Topic = Topic,
Offset = (int)message.Meta.Offset,
Partition = message.Meta.PartitionId,
MessageContent = MessageContent,
CreatedAt = DateTime.UtcNow
};
KafkaRepo.InsertKafkaConsumerMessage(ConsumerMessage);
KafkaRepo.Dispose();
}
}
