public KafkaConsumer(string brokerList,
string[] topics,
string groupId,
string consumerId,
OnKafkaMessageReceived <TKey, TValue> onMessageReceived,
ConsumerConfig consumerConfig = null)
: base(topics, groupId, consumerId, consumerConfig)
{
BrokerList = brokerList;
if (string.IsNullOrWhiteSpace(brokerList))
{
throw new ArgumentException("Value cannot be null or whitespace.", nameof(brokerList));
}
if (string.IsNullOrWhiteSpace(groupId))
{
throw new ArgumentException("Value cannot be null or whitespace.", nameof(groupId));
}
OnMessageReceived = onMessageReceived;
ConsumerConfiguration = new Confluent.Kafka.ConsumerConfig
{
GroupId = GroupId,
ClientId = consumerId,
EnableAutoCommit = false,
//{"socket.blocking.max.ms", ConsumerConfig["socket.blocking.max.ms"] ?? 50},
//{"fetch.error.backoff.ms", ConsumerConfig["fetch.error.backoff.ms"] ?? 50},
SocketNagleDisable = true,
//{"statistics.interval.ms", 60000},
//{"retry.backoff.ms", ConsumerConfig.BackOffIncrement.ToString()},
BootstrapServers = BrokerList,
AutoOffsetReset = (Confluent.Kafka.AutoOffsetReset)ConsumerConfig.AutoOffsetReset
};
}
public MessageService(Guid consumerID, string topic, string group) : base(consumerID, topic, group)
{
ConsumerConfig = new CKafka.ConsumerConfig
{
GroupId = group,
BootstrapServers = KafkaEndpoint,
AutoOffsetReset = CKafka.AutoOffsetReset.Earliest,
};
}
private IConsumer <string, string> GetConsumer(string broker, string group)
{
var consumerConfiguration = new ConsumerConfig
{
GroupId = group,
ClientId = $"KafkaTools.{group}",
EnableAutoCommit = false,
//{"socket.blocking.max.ms", ConsumerConfig["socket.blocking.max.ms"] ?? 50},
//{"fetch.error.backoff.ms", ConsumerConfig["fetch.error.backoff.ms"] ?? 50},
SocketNagleDisable = true,
//{"statistics.interval.ms", 60000},
BootstrapServers = broker
};
return(new ConsumerBuilder <string, string>(consumerConfiguration).Build());
}
// Metoda Subscribe omogucava povezivanje Consumera sa odredjenim Topicom i particijom u okviru tog Topica
public override void Subscribe(string topic, string group)
{
try
{
// Za konfiguraciju je dovoljno navesti particiju (GroupId), kafka Endpoint (BootstrapServers) i
// Nacin citanja poruka u zavisnosti od Offset-a
ConsumerConfig = new CKafka.ConsumerConfig
{
GroupId = group,
BootstrapServers = KafkaEndpoint,
AutoOffsetReset = CKafka.AutoOffsetReset.Earliest,
};
// Kreira se Consumer pri cemu se navodi tip podatka poruke koji ce prihvatati
ConsumerBuild = new CKafka.ConsumerBuilder <CKafka.Ignore, string>(ConsumerConfig).Build();
// Povezivanje na jedan ili vise Topic-a
ConsumerBuild.Subscribe(new List <string>()
{
topic
});
Console.WriteLine("MessageService ==> Consumer: " + group + " subscribed to topic: " + topic);
cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
// Pozivanje metode za osluskivanje novih dogadjaja u novom thread-u kako bi se omogucilo
// opsluzivanje bez blokiranja veceg broja razlicitih Consumer servisa
RunMethodInSeparateThread(MessageConsume);
}
catch (CKafka.KafkaException ex)
{
Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message from topic: " + topic);
Console.WriteLine(ex.Message);
throw new Exception("Failed to consume message", ex);
}
catch (Exception ex)
{
Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message from topic: " + topic);
Console.WriteLine(ex.Message);
throw new Exception("Failed to consume message", ex);
}
}
// Metoda koja se koristi prilikom slanja GET zahteva Messaging kontroleru
// Sluzi da dobavi listu poruka koju je neki servis propustio u toku odredjenog vremena tako sto ce
// iscitati sve poruke iz Message Queue-a
public override List <string> Consume(string topic, string group)
{
try
{
List <string> returnList = new List <string>();
// U konfiguraciji Consumera se navode svi parametri kako bi se iscitavanje izvrsilo neometano
// GroupId je particija kojoj se pristupa, BootstrapServers je Endpoint na kojem se kafka nalazi
// U okviru konfiguracije navodi se jos nacin na koji cete citati poruke (AutoOffsetReset i EnablePartitionEof),
// Da li zelite da se Topici automatski kreiraju ukoliko ne postoje
ConsumerConfig = new CKafka.ConsumerConfig
{
GroupId = group,
BootstrapServers = KafkaEndpoint,
AutoOffsetReset = CKafka.AutoOffsetReset.Earliest,
AllowAutoCreateTopics = true,
EnablePartitionEof = EnableParttitonEof
};
using (var consumerBulid = new CKafka.ConsumerBuilder <CKafka.Ignore, string>(ConsumerConfig).Build())
{
// Consumer se povezuje metodom Subscribe jedan ili vise Topic-a
consumerBulid.Subscribe(new List <string>()
{
topic
});
// Definisanje Console Key-a za prekid komunikacije
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
CKafka.ConsumeResult <CKafka.Ignore, string> cr;
Console.WriteLine("Consumed messages: ");
do
{
// Citanje poruka pozivom metode Consume koja je definisana za Consumera
cr = consumerBulid.Consume();
// Citanje poruka vrsi se sekvencijalno, dokle god se ne dodje do kraja Message Queue-a
if (!cr.IsPartitionEOF)
{
Console.WriteLine($"Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'.");
returnList.Add(cr.Message.Value);
}
else
{
try
{
// Kada se dodje do kraja Message Queue-a, metodom Commit se pomera Offset na kraj
consumerBulid.Commit();
}
catch (CKafka.KafkaException ex)
{
Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message: " + cr.Message.Value + " from topic: " + topic);
Console.WriteLine(ex.Message);
continue;
}
}
} while (!cr.IsPartitionEOF);
}
return(returnList);
}
catch (Exception ex)
{
Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message from topic: " + topic);
Console.WriteLine(ex.Message);
return(new List <string>());
}
}
