public KafkaConsumer(string brokerList, string[] topics, string groupId, string consumerId, OnKafkaMessageReceived <TKey, TValue> onMessageReceived, ConsumerConfig consumerConfig = null) : base(topics, groupId, consumerId, consumerConfig) { BrokerList = brokerList; if (string.IsNullOrWhiteSpace(brokerList)) { throw new ArgumentException("Value cannot be null or whitespace.", nameof(brokerList)); } if (string.IsNullOrWhiteSpace(groupId)) { throw new ArgumentException("Value cannot be null or whitespace.", nameof(groupId)); } OnMessageReceived = onMessageReceived; ConsumerConfiguration = new Confluent.Kafka.ConsumerConfig { GroupId = GroupId, ClientId = consumerId, EnableAutoCommit = false, //{"socket.blocking.max.ms", ConsumerConfig["socket.blocking.max.ms"] ?? 50}, //{"fetch.error.backoff.ms", ConsumerConfig["fetch.error.backoff.ms"] ?? 50}, SocketNagleDisable = true, //{"statistics.interval.ms", 60000}, //{"retry.backoff.ms", ConsumerConfig.BackOffIncrement.ToString()}, BootstrapServers = BrokerList, AutoOffsetReset = (Confluent.Kafka.AutoOffsetReset)ConsumerConfig.AutoOffsetReset }; }
public MessageService(Guid consumerID, string topic, string group) : base(consumerID, topic, group) { ConsumerConfig = new CKafka.ConsumerConfig { GroupId = group, BootstrapServers = KafkaEndpoint, AutoOffsetReset = CKafka.AutoOffsetReset.Earliest, }; }
private IConsumer <string, string> GetConsumer(string broker, string group) { var consumerConfiguration = new ConsumerConfig { GroupId = group, ClientId = $"KafkaTools.{group}", EnableAutoCommit = false, //{"socket.blocking.max.ms", ConsumerConfig["socket.blocking.max.ms"] ?? 50}, //{"fetch.error.backoff.ms", ConsumerConfig["fetch.error.backoff.ms"] ?? 50}, SocketNagleDisable = true, //{"statistics.interval.ms", 60000}, BootstrapServers = broker }; return(new ConsumerBuilder <string, string>(consumerConfiguration).Build()); }
// Metoda Subscribe omogucava povezivanje Consumera sa odredjenim Topicom i particijom u okviru tog Topica public override void Subscribe(string topic, string group) { try { // Za konfiguraciju je dovoljno navesti particiju (GroupId), kafka Endpoint (BootstrapServers) i // Nacin citanja poruka u zavisnosti od Offset-a ConsumerConfig = new CKafka.ConsumerConfig { GroupId = group, BootstrapServers = KafkaEndpoint, AutoOffsetReset = CKafka.AutoOffsetReset.Earliest, }; // Kreira se Consumer pri cemu se navodi tip podatka poruke koji ce prihvatati ConsumerBuild = new CKafka.ConsumerBuilder <CKafka.Ignore, string>(ConsumerConfig).Build(); // Povezivanje na jedan ili vise Topic-a ConsumerBuild.Subscribe(new List <string>() { topic }); Console.WriteLine("MessageService ==> Consumer: " + group + " subscribed to topic: " + topic); cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; // Pozivanje metode za osluskivanje novih dogadjaja u novom thread-u kako bi se omogucilo // opsluzivanje bez blokiranja veceg broja razlicitih Consumer servisa RunMethodInSeparateThread(MessageConsume); } catch (CKafka.KafkaException ex) { Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message from topic: " + topic); Console.WriteLine(ex.Message); throw new Exception("Failed to consume message", ex); } catch (Exception ex) { Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message from topic: " + topic); Console.WriteLine(ex.Message); throw new Exception("Failed to consume message", ex); } }
// Metoda koja se koristi prilikom slanja GET zahteva Messaging kontroleru // Sluzi da dobavi listu poruka koju je neki servis propustio u toku odredjenog vremena tako sto ce // iscitati sve poruke iz Message Queue-a public override List <string> Consume(string topic, string group) { try { List <string> returnList = new List <string>(); // U konfiguraciji Consumera se navode svi parametri kako bi se iscitavanje izvrsilo neometano // GroupId je particija kojoj se pristupa, BootstrapServers je Endpoint na kojem se kafka nalazi // U okviru konfiguracije navodi se jos nacin na koji cete citati poruke (AutoOffsetReset i EnablePartitionEof), // Da li zelite da se Topici automatski kreiraju ukoliko ne postoje ConsumerConfig = new CKafka.ConsumerConfig { GroupId = group, BootstrapServers = KafkaEndpoint, AutoOffsetReset = CKafka.AutoOffsetReset.Earliest, AllowAutoCreateTopics = true, EnablePartitionEof = EnableParttitonEof }; using (var consumerBulid = new CKafka.ConsumerBuilder <CKafka.Ignore, string>(ConsumerConfig).Build()) { // Consumer se povezuje metodom Subscribe jedan ili vise Topic-a consumerBulid.Subscribe(new List <string>() { topic }); // Definisanje Console Key-a za prekid komunikacije CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; CKafka.ConsumeResult <CKafka.Ignore, string> cr; Console.WriteLine("Consumed messages: "); do { // Citanje poruka pozivom metode Consume koja je definisana za Consumera cr = consumerBulid.Consume(); // Citanje poruka vrsi se sekvencijalno, dokle god se ne dodje do kraja Message Queue-a if (!cr.IsPartitionEOF) { Console.WriteLine($"Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'."); returnList.Add(cr.Message.Value); } else { try { // Kada se dodje do kraja Message Queue-a, metodom Commit se pomera Offset na kraj consumerBulid.Commit(); } catch (CKafka.KafkaException ex) { Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message: " + cr.Message.Value + " from topic: " + topic); Console.WriteLine(ex.Message); continue; } } } while (!cr.IsPartitionEOF); } return(returnList); } catch (Exception ex) { Console.WriteLine("MessageService ==> Consumer: " + group + " failed to consume message from topic: " + topic); Console.WriteLine(ex.Message); return(new List <string>()); } }