protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var kafkaConsumer = default(IConsumer <int, MyMessage>); try { var config = new ConsumerConfig { BootstrapServers = string.Join(',', bootstrapServers), // In order to guarantee only processed messages are committed, auto commit // must be disabled. EnableAutoCommit = false, // The following settings are just for testing purposes to allow us to run the // consumer multiple times without having to produce new messages every time. GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest, }; kafkaConsumer = new ConsumerBuilder <int, MyMessage>(config) .SetValueDeserializer(new JsonSerializer <MyMessage>()) .SetPartitionsAssignedHandler(OnPartitionAssigned) .SetPartitionsRevokedHandler(OnPartitionRevoked) .Build(); kafkaConsumer.Subscribe("resilience-tests"); var consumer = new MyConsumer(kafkaConsumer, this.processor); do { stoppingToken.ThrowIfCancellationRequested(); await consumer.ConsumeAsync(stoppingToken); }while (true); } catch (OperationCanceledException) { try { kafkaConsumer?.Close(); } catch { // Ignore close exceptions. } } catch (Exception ex) { Console.WriteLine(ex); } }
public static void Run(IServiceProvider serviceProvider) { // get exchange settings var settings = serviceProvider.GetRequiredService <IOptions <ExchangeSettings> >().Value; // get the user manager & email sender var userManager = serviceProvider.GetRequiredService <UserManager <ApplicationUser> >(); var context = serviceProvider.GetRequiredService <ApplicationDbContext>(); var emailSender = serviceProvider.GetRequiredService <IEmailSender>(); var conf = new ConsumerConfig { GroupId = "order-updates-group", BootstrapServers = settings.KafkaHost, // Note: The AutoOffsetReset property determines the start offset in the event // there are not yet any committed offsets for the consumer group for the // topic/partitions of interest. By default, offsets are committed // automatically, so in this example, consumption will only start from the // earliest message in the topic 'my-topic' the first time you run the program. AutoOffsetReset = AutoOffsetReset.Earliest, }; using (var c = new ConsumerBuilder <Ignore, string>(conf).Build()) { c.Subscribe(new string[] { OrdersTopic, DealsTopic }); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { while (true) { try { var cr = c.Consume(cts.Token); Console.WriteLine($"Consumed message at: '{cr.TopicPartitionOffset}'."); switch (cr.Topic) { case OrdersTopic: ProcessOrder(context, userManager, settings, emailSender, cr.Message.Value); break; case DealsTopic: ProcessDeal(context, userManager, emailSender, cr.Message.Value); break; default: break; } // commit queue offsets c.Commit(cr); } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. c.Close(); } } }
public void Consume(string bootstrapServers, string schemaRegistryUrl, string topicName) { var schemaRegistryRequestTimeoutMs = 30000; var schemaRegistryMaxCachedSchemas = 1000; var groupID = Guid.NewGuid().ToString(); var bufferBytes = 1024; var autoRegisterSchema = true; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryUrl, // optional schema registry client properties: SchemaRegistryRequestTimeoutMs = schemaRegistryRequestTimeoutMs, SchemaRegistryMaxCachedSchemas = schemaRegistryMaxCachedSchemas }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, AutoOffsetReset = AutoOffsetReset.Earliest, GroupId = groupID // "Test" //Guid.NewGuid().ToString() }; var avroSerializerConfig = new AvroSerializerConfig { // optional Avro serializer properties: BufferBytes = bufferBytes, AutoRegisterSchemas = autoRegisterSchema }; NewConstructionAddressEvent addr = new NewConstructionAddressEvent(); using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var consumer = new ConsumerBuilder <string, NewConstructionAddressEvent>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync()) .SetValueDeserializer(new AvroDeserializer <NewConstructionAddressEvent>(schemaRegistry).AsSyncOverAsync()) .SetErrorHandler((_, e) => logger.LogError($"Error: {e.Reason}")) .Build()) { try { logger.LogInformation($"Starting consumer.subscribe."); consumer.Subscribe(topicName); while (true) { try { logger.LogInformation($"Starting: consumer.Consume"); var consumeResult = consumer.Consume(); string k = consumeResult.Key; logger.LogInformation($"BusMessage: {consumeResult.Message}, constructionAddressId: {consumeResult.Value.constructionAddressId}"); } catch (OperationCanceledException) { logger.LogInformation($"OperationCancelled for consumer.Consume"); break; } catch (ConsumeException e) { logger.LogInformation(e, $"Consume error: {e.Error.Reason}"); break; } } } catch (Exception ex) { logger.LogError(ex, $"Consume error: {ex.Message}"); } finally { consumer.Close(); } } }
public void Consume() { var config = new ConsumerConfig { BootstrapServers = _kafkaSettings.BrokerList, GroupId = _kafkaSettings.GroupId, //"csharp-consumer", EnableAutoCommit = false, StatisticsIntervalMs = 5000, SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; const int commitPeriod = 5; // Note: If a key or value deserializer is not set (as is the case below), the // deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers // will be used automatically (where available). The default deserializer for string // is UTF8. The default deserializer for Ignore returns null for all input data // (including non-null data). using (var consumer = new ConsumerBuilder <Ignore, string>(config) // Note: All handlers are called on the main .Consume thread. .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}")) .SetPartitionsAssignedHandler((c, partitions) => { Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}]"); // possibly manually specify start offsets or override the partition assignment provided by // the consumer group by returning a list of topic/partition/offsets to assign to, e.g.: // // return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp])); }) .SetPartitionsRevokedHandler((c, partitions) => { Console.WriteLine($"Revoking assignment: [{string.Join(", ", partitions)}]"); }) .Build()) { consumer.Subscribe(_kafkaSettings.Topics); try { while (true) { try { var consumeResult = consumer.Consume(_kafkaSettings.CancellationToken); if (consumeResult.IsPartitionEOF) { Console.WriteLine( $"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}."); continue; } Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Value}"); var consumed = ReceiveMessage?.Invoke(consumeResult.Value.ConvertToMessage(), _kafkaSettings.CancellationToken); if ((consumed.HasValue && consumed.Value) && consumeResult.Offset % commitPeriod == 0) { // The Commit method sends a "commit offsets" request to the Kafka // cluster and synchronously waits for the response. This is very // slow compared to the rate at which the consumer is capable of // consuming messages. A high performance application will typically // commit offsets relatively infrequently and be designed handle // duplicate messages in the event of failure. try { consumer.Commit(consumeResult); } catch (KafkaException e) { Console.WriteLine($"Commit error: {e.Error.Reason}"); } } } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { Console.WriteLine("Closing consumer."); consumer.Close(); } } }
public async Task Subscribe <T>(Action <T> callback) where T : IEvent { var conf = new ConsumerConfig { GroupId = $"{Dns.GetHostName()}-{Assembly.GetExecutingAssembly()}", BootstrapServers = _parameters.GetStringParameter("KAFKA_BOOTSTRAP_SERVERS"), AutoOffsetReset = AutoOffsetReset.Earliest }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = _parameters.GetStringParameter("KAFKA_SCHEMA_REGISTRY_URLS") }; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var c = new ConsumerBuilder <Ignore, T>(conf) .SetValueDeserializer(new AvroDeserializer <T>(schemaRegistry).AsSyncOverAsync()) .SetErrorHandler((_, e) => _logger.LogError($"Exception has occured when trying to consume an event : {e.Reason}")) .Build()) { c.Subscribe(GetTopicName <T>()); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; await Task.Factory.StartNew(() => { try { while (true) { try { var cr = c.Consume(cts.Token); if (cr.Message.Value == null) { continue; } _logger.LogDebug($"Consumed message '{cr.Message.Value}' at: '{cr.TopicPartitionOffset}'."); try { callback(cr.Message.Value); } catch (Exception e) { _logger.LogError($"Event {cr.Message.Value} was not processed \n\a" + e.GetFullMessage()); } } catch (ConsumeException e) { _logger.LogError($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { c.Close(); } }, cts.Token); } }
public async Task <EventsData> GetSampleEvents(int seconds) { var config = new ConsumerConfig { BootstrapServers = _brokerList, GroupId = _consumerGroup, EnableAutoCommit = false, SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Latest }; // Set the authentication for EventHub Kafka if (_inputType == Constants.InputType_KafkaEventHub) { config.SecurityProtocol = SecurityProtocol.SaslSsl; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = _connectionString; config.SslCaLocation = _cacertLocation; } StartTimer(seconds); const int commitPeriod = 5; using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build()) { EventsData eventsData = new EventsData(); consumer.Subscribe(_topics); try { while (true) { try { var consumeResult = consumer.Consume(new TimeSpan(0, 0, seconds)); if (_timeout) { _logger.LogInformation($"Closing consumer"); consumer.Close(); return(await Task.FromResult(eventsData).ConfigureAwait(false)); } if (consumeResult.IsPartitionEOF) { _logger.LogInformation($"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}."); continue; } _logger.LogInformation($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Value}"); // Get raw data EventRaw er = new EventRaw { Raw = consumeResult.Value, Properties = new Dictionary <string, string>() { { "HeadersCount", consumeResult.Headers.Count.ToString() } }, }; // Set properties (using the Headers) if (consumeResult.Headers != null && consumeResult.Headers.Count > 0) { for (int i = 0; i < consumeResult.Headers.Count; i++) { string key = consumeResult.Headers[i].Key; string val = System.Text.Encoding.UTF8.GetString(consumeResult.Headers[i].GetValueBytes()); er.Properties.Add(key, val); } } // Set the SystemProperties er.SystemProperties.Add("Topic", consumeResult.Topic); er.SystemProperties.Add("Partition", consumeResult.Partition.Value.ToString()); er.SystemProperties.Add("Offset", consumeResult.Offset.Value.ToString()); er.SystemProperties.Add("UtcDateTime", consumeResult.Timestamp.UtcDateTime.ToString()); er.SystemProperties.Add("UnixTimestampMs", consumeResult.Timestamp.UnixTimestampMs.ToString()); er.Json = JsonConvert.SerializeObject(er); eventsData.EventsJson += er.Json + "\r\n"; eventsData.Events.Add(er); if (consumeResult.Offset % commitPeriod == 0) { try { consumer.Commit(consumeResult); } catch (KafkaException e) { _logger.LogError($"Commit error: {e.Error.Reason}\n{e.ToString()}"); } } } catch (ConsumeException e) { _logger.LogError($"Consume error: {e.Error.Reason}\n{e.ToString()}"); } } } catch (OperationCanceledException e) { _logger.LogInformation($"Closing consumer"); consumer.Close(); return(await Task.FromResult(eventsData).ConfigureAwait(false)); } } }
private static void Main(string[] args) { var config = new ConsumerConfig { GroupId = "messages-consumers", BootstrapServers = "localhost:9092", AutoOffsetReset = AutoOffsetReset.Earliest }; Console.Write("Would you like to see message content? Y/N: "); var selection = Console.ReadLine(); var result = string.Empty; switch (selection) { case "Y": result = selection; break; case "N": result = selection; break; default: Console.WriteLine("Invalid input. Messages content no will be shown."); result = "N"; break; } Console.WriteLine(); Console.WriteLine("Press Ctrl+C to exit."); Console.WriteLine("Listening..."); Console.WriteLine(); using (var consumer = new ConsumerBuilder <Ignore, byte[]>(config).Build()) { consumer.Subscribe(topicName); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; try { while (true) { try { var message = consumer.Consume(cts.Token); if (result == "Y") { Console.WriteLine($"{message.Message.Timestamp.UtcDateTime.ToLocalTime()}: " + $"{Serializer.DeserializeMessage(message.Message.Value)}"); Console.WriteLine(); } else { Console.WriteLine($"{message.Message.Timestamp.UtcDateTime.ToLocalTime()}: message received."); } } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { consumer.Close(); } } Console.WriteLine("Press any key to exit..."); Console.ReadKey(); }
public static void Main(string[] args) { provider.FromXmlString(File.ReadAllText("C:\\Projects\\keys\\key.xml")); //// buffer to write byte sequence of the given block_size blockSize = (provider.KeySize / 8); rijAlg.Mode = CipherMode.ECB; rijAlg.Padding = PaddingMode.ISO10126; var conf = new ConsumerConfig { GroupId = "test-consumer-group", BootstrapServers = "localhost:9093", StatisticsIntervalMs = 10000, // QueuedMinMessages = 1000, FetchWaitMaxMs = 5, FetchErrorBackoffMs = 10, EnableAutoCommit = true, SocketReceiveBufferBytes = 1048576, SecurityProtocol = SecurityProtocol.Ssl, EnableSslCertificateVerification = true, SslCaLocation = "C:\\Projects\\keys\\cert-signed", // Debug="consumer,topic", AutoCommitIntervalMs = 10000, // Note: The AutoOffsetReset property determines the start offset in the event // there are not yet any committed offsets for the consumer group for the // topic/partitions of interest. By default, offsets are committed // automatically, so in this example, consumption will only start from the // earliest message in the topic 'my-topic' the first time you run the program. // AutoOffsetReset = AutoOffsetReset.Earliest }; var commitPeriod = 10; // in ms ( for manual commit operation ) ThreadPool.SetMinThreads(8, 5); ThreadPool.SetMaxThreads(10, 10); using (var c = new ConsumerBuilder <Ignore, string>(conf) .SetStatisticsHandler((_, json) => { if (totalMessages > 0 && !isTestingInProgress) { var ellapsedTime = ((statisticsEndTime < lastMessageReceivedTime ? lastMessageReceivedTime : statisticsEndTime) - statisticsStartTime).TotalMilliseconds; Console.WriteLine($"Statistics: Aggregate time ellapsed - {totalEllapsed}ms, Total Messages - {totalMessages}, Average response - {totalEllapsed / totalMessages}ms"); Console.WriteLine($"Statistics: Total time elapsed - {ellapsedTime}, Throughput - {totalMessages / (ellapsedTime / 1000)} messages/sec.,"); Console.WriteLine($"Statistics: Total messages above {maxDelayThreshold}ms - {totalMessageAboveThreshold}"); Console.WriteLine($"Last mesasge received time - {lastMessageReceivedTime}, Statistics ended time - {statisticsEndTime}"); Console.WriteLine($"Maximum delay - {maxDelay}ms"); totalEllapsed = 0; totalMessages = 0; totalMessageAboveThreshold = 0; maxDelay = 0; } }) .Build()) { c.Subscribe("my-topic"); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { while (true) { try { var cr = c.Consume(cts.Token); if (cr.IsPartitionEOF) { Console.WriteLine( $"Reached end of topic {cr.Topic}, partition {cr.Partition}, offset {cr.Offset}."); continue; } var data = cr.Message.Value; ThreadPool.QueueUserWorkItem(ThreadExecutionHandler, data); lastMessageReceivedTime = DateTime.Now; //ThreadExecutionHandler(data); // Console.WriteLine($"Consumed message at: '{cr.TopicPartitionOffset}'."); //if (cr.Offset % commitPeriod == 0) //{ // try // { // c.Commit(cr); // } // catch (Exception ex) // { // Console.WriteLine(ex); // } //} } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. c.Close(); } } }
protected void Run(string topics, int concurrence, CancellationToken cancellationToken) { int nPart = -1; for (int i = 0; i < concurrence; i++) { var ts = new ThreadStart(() => { using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).Build()) { consumer.Subscribe(topics); while (!cancellationToken.IsCancellationRequested) { ConsumeResult <Ignore, string> consumeResult = null; try { consumeResult = consumer.Consume(1000); // Loggar partições conectadas if (consumer.Assignment.Count != nPart) { nPart = consumer.Assignment.Count; consumer.Assignment.ForEach((tp) => { logger.LogDebug($"Assinado para partição [{tp.Partition}] para Thread [{Thread.CurrentThread.ManagedThreadId}]"); }); } if (consumeResult == null) { continue; } this.Handler(consumeResult); } catch (Exception ex) { logger.LogError(ex.Message); //Retry //DLQ Thread.Sleep(1000); } finally { if (consumeResult != null) { consumer.Commit(consumeResult); } } } consumer.Close(); } }); new Thread(ts).Start(); } }
public override async Task Read(Dictionary <string, string> properties) { var groupId = EndPointConfig.Properties["GroupId"]; var servers = EndPointConfig.Properties["BootstrapServers"]; var topic = EndPointConfig.Properties["Topic"]; var topics = new List <string>() { topic }; var config = new ConsumerConfig { BootstrapServers = servers, GroupId = groupId, EnableAutoCommit = false, StatisticsIntervalMs = 5000, SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; const int commitPeriod = 5; // Note: If a key or value deserializer is not set (as is the case below), the // deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers // will be used automatically (where available). The default deserializer for string // is UTF8. The default deserializer for Ignore returns null for all input data // (including non-null data). using (var consumer = new ConsumerBuilder <Ignore, string>(config) // Note: All handlers are called on the main .Consume thread. .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}")) .SetPartitionsAssignedHandler((c, partitions) => { _logger.Info($"Assigned partitions: [{0}]", string.Join(", ", partitions)); // possibly manually specify start offsets or override the partition assignment provided by // the consumer group by returning a list of topic/partition/offsets to assign to, e.g.: // // return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp])); }) .SetPartitionsRevokedHandler((c, partitions) => { _logger.Info($"Revoking assignment: [{0}]", string.Join(", ", partitions)); }) .Build()) { consumer.Subscribe(topics); try { while (!Token.IsCancellationRequested) { try { var consumeResult = consumer.Consume(Token); if (consumeResult.IsPartitionEOF) { _logger.Info($"Reached end of topic {0}, partition {1}, offset {2}.", consumeResult.Topic, consumeResult.Partition, consumeResult.Offset); continue; } _logger.Info($"Received message at {0}: {1}", consumeResult.TopicPartitionOffset, consumeResult.Value); var nameValues = (Dictionary <string, string>)JsonConvert.DeserializeObject(consumeResult.Value, typeof(Dictionary <string, string>)); var entityCollection = new EntityCollection(); foreach (var key in nameValues.Keys) { entityCollection.Entities.Add(key, nameValues[key]); } Data.Add(entityCollection); await SignalHandler(new Dictionary <string, string>()); if (consumeResult.Offset % commitPeriod == 0) { // The Commit method sends a "commit offsets" request to the Kafka // cluster and synchronously waits for the response. This is very // slow compared to the rate at which the consumer is capable of // consuming messages. A high performance application will typically // commit offsets relatively infrequently and be designed handle // duplicate messages in the event of failure. try { consumer.Commit(consumeResult); } catch (KafkaException e) { _logger.Error($"Commit error: {0}", e.Error.Reason); } } } catch (ConsumeException e) { _logger.Error($"Consume error: {0}", e.Error.Reason); } } } catch (OperationCanceledException) { _logger.Debug("Closing consumer."); consumer.Close(); } } }
public static void ProduceConsume(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; var adminClientConfig = new AdminClientConfig { BootstrapServers = bootstrapServers }; string topic = Guid.NewGuid().ToString(); using (var adminClient = new AdminClientBuilder(adminClientConfig).Build()) { adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, User>(producerConfig) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(new AvroSerializer <User>(schemaRegistry)) .Build()) { for (int i = 0; i < 100; ++i) { var user = new User { name = i.ToString(), favorite_number = i, favorite_color = "blue" }; producer .ProduceAsync(topic, new Message <string, User> { Key = user.name, Value = user }) .Wait(); } Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var consumer = new ConsumerBuilder <string, User>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync()) .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry).AsSyncOverAsync()) .SetErrorHandler((_, e) => Assert.True(false, e.Reason)) .Build()) { consumer.Subscribe(topic); int i = 0; while (true) { var record = consumer.Consume(TimeSpan.FromMilliseconds(100)); if (record == null) { continue; } if (record.IsPartitionEOF) { break; } Assert.Equal(i.ToString(), record.Message.Key); Assert.Equal(i.ToString(), record.Message.Value.name); Assert.Equal(i, record.Message.Value.favorite_number); Assert.Equal("blue", record.Message.Value.favorite_color); i += 1; } Assert.Equal(100, i); consumer.Close(); } }
private void Subscribe() { if (init == 1) { return; } if (_option.ConsumerConfig == null) { throw new ArgumentNullException(nameof(_option.ConsumerConfig), nameof(KafkaOptions)); } if (Interlocked.Exchange(ref init, 1) == 1) { return; } try { _log.LogInformation($"Subscribe to topic {_option.ResponseTopic}"); Task.Run(async() => { using var consumer = new ConsumerBuilder <byte[], byte[]>(_option.ConsumerConfig).Build(); consumer.Subscribe(_option.ResponseTopic); try { while (true) { var message = consumer.Consume(_cancellationTokenSource.Token); try { await Handle(message); } catch (Exception e) { _log.LogError(e, $"Unprocessed exception,topic: {message.Topic}, partition: {message.Partition.Value} offset: {message.Offset.Value}"); } } } catch (OperationCanceledException e) { _log.LogInformation(e, "Kafka client cancelled. Close consumer"); consumer.Close(); } }, _cancellationTokenSource.Token).ContinueWith(task => { if (task.Exception != null) { _log.LogError(task.Exception, "Error subscribe kafka "); } else { _log.LogInformation($"Kafka client subscribe started. {task.Status}"); } }, _cancellationTokenSource.Token); } catch (Exception) { Interlocked.Exchange(ref init, 0); throw; } }
public override void Run(string brokerList, List <string> topics, CancellationToken cancellationToken) { var config = new ConsumerConfig { // the group.id property must be specified when creating a consumer, even // if you do not intend to use any consumer group functionality. GroupId = new Guid().ToString(), BootstrapServers = brokerList, // partition offsets can be committed to a group even by consumers not // subscribed to the group. in this example, auto commit is disabled // to prevent this from occurring. EnableAutoCommit = true }; Stopwatch sw = new Stopwatch(); sw.Start(); Console.WriteLine($"sw.ElapsedMilliseconds start"); using (var consumer = new ConsumerBuilder <int, string>(config) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { consumer.Assign(topics.Select(topic => new TopicPartitionOffset(topic, 0, Offset.Beginning)).ToList()); try { while (true) { try { var result = consumer.ConsumeBatch <int, string>(1000, 10); if (result.Messages.Count > 0) { var connString = Utility.GetConnectionString("ConnectionStrings:DefaultConnectionEF"); using var conn = new NpgsqlConnection(connString); conn.Open(); var valuesTableSql = string.Join(",", Enumerable.Range(0, result.Messages.Count).Select(i => $"(@p1{i}, @p2{i},@p3{i},@p4{i},@p5{i})")); using (var tx = conn.BeginTransaction()) { using (var cmd = new NpgsqlCommand($"INSERT INTO \"Messages\" (\"Id\", \"Created\", \"Content\", \"IsReceived\", \"ReceivedTimestamp\") VALUES {valuesTableSql};", conn)) { for (int i = 0; i < result.Messages.Count; ++i) { cmd.Parameters.AddWithValue($"p1{i}", result.Messages.ElementAt(i).Key); cmd.Parameters.AddWithValue($"p2{i}", DateTime.Now); cmd.Parameters.AddWithValue($"p3{i}", result.Messages.ElementAt(i).Value); cmd.Parameters.AddWithValue($"p4{i}", false); cmd.Parameters.AddWithValue($"p5{i}", DateTime.Now); } cmd.ExecuteNonQuery(); } tx.Commit(); } Console.WriteLine($"after database: {sw.ElapsedMilliseconds}, MC: {result.MessagesConsumed}, WM: {result.BatchWindowMilliseconds}"); } } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { Console.WriteLine("Closing consumer."); consumer.Close(); } } sw.Stop(); Console.WriteLine($"sw.ElapsedMilliseconds: {sw.ElapsedMilliseconds}"); }
void Listen() { var config = new ConsumerConfig { BootstrapServers = BrokerList, GroupId = GroupId, //"csharp-consumer", EnableAutoCommit = false, StatisticsIntervalMs = 5000, SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true, // A good introduction to the CooperativeSticky assignor and incremental rebalancing: // https://www.confluent.io/blog/cooperative-rebalancing-in-kafka-streams-consumer-ksqldb/ PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky }; const int commitPeriod = 1; // Note: If a key or value deserializer is not set (as is the case below), the // deserializer corresponding to the appropriate type from Confluent.Kafka.Deserializers // will be used automatically (where available). The default deserializer for string // is UTF8. The default deserializer for Ignore returns null for all input data // (including non-null data). using (var consumer = new ConsumerBuilder <Ignore, string>(config) // Note: All handlers are called on the main .Consume thread. .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) //.SetStatisticsHandler((_, json) => Console.WriteLine($"Statistics: {json}")) .SetPartitionsAssignedHandler((c, partitions) => { // Since a cooperative assignor (CooperativeSticky) has been configured, the // partition assignment is incremental (adds partitions to any existing assignment). Console.WriteLine($"Incremental partition assignment: [{string.Join(", ", partitions)}]"); // Possibly manually specify start offsets by returning a list of topic/partition/offsets // to assign to, e.g.: // return partitions.Select(tp => new TopicPartitionOffset(tp, externalOffsets[tp])); }) .SetPartitionsRevokedHandler((c, partitions) => { // Since a cooperative assignor (CooperativeSticky) has been configured, the revoked // assignment is incremental (may remove only some partitions of the current assignment). Console.WriteLine($"Incremental partition revokation: [{string.Join(", ", partitions)}]"); }) .SetPartitionsLostHandler((c, partitions) => { // The lost partitions handler is called when the consumer detects that it has lost ownership // of its assignment (fallen out of the group). Console.WriteLine($"Partitions were lost: [{string.Join(", ", partitions)}]"); }) .Build()) { consumer.Subscribe(Topics); try { while (true) { try { var consumeResult = consumer.Consume(CancellationToken); if (consumeResult.IsPartitionEOF) { Console.WriteLine( $"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}."); continue; } OnMessageReceived(consumeResult.Message.Value); Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Message.Value}"); if (consumeResult.Offset % commitPeriod == 0) { // The Commit method sends a "commit offsets" request to the Kafka // cluster and synchronously waits for the response. This is very // slow compared to the rate at which the consumer is capable of // consuming messages. A high performance application will typically // commit offsets relatively infrequently and be designed handle // duplicate messages in the event of failure. try { consumer.Commit(consumeResult); } catch (KafkaException e) { Console.WriteLine($"Commit error: {e.Error.Reason}"); } } } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { Console.WriteLine("Closing consumer."); consumer.Close(); } } }
public Task Process(CancellationToken cancellationToken) { const int commitPeriod = 5; var config = new ConsumerConfig { BootstrapServers = "127.0.0.1:9092", GroupId = "test-consumer" }; // Create the consumer using (var consumer = new ConsumerBuilder <Null, string>(config).Build()) { Console.WriteLine($"Subscribing to Kafka Topic {"testtopic"}"); // Subscribe to the Kafka topic consumer.Subscribe(new List <string>() { "testtopic" }); try { while (true) { try { var consumeResult = consumer.Consume(cancellationToken); if (consumeResult.IsPartitionEOF) { Console.WriteLine( $"Reached end of topic {consumeResult.Topic}, partition {consumeResult.Partition}, offset {consumeResult.Offset}."); continue; } Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: {consumeResult.Message.Value}"); if (consumeResult.Offset % commitPeriod == 0) { try { consumer.Commit(consumeResult); } catch (KafkaException e) { Console.WriteLine($"Commit error: {e.Error.Reason}"); } } } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { Console.WriteLine("Closing consumer."); consumer.Close(); } } return(Task.Run(() => true)); }
public static void Consumer_AutoCommit(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start Consumer_AutoCommit"); int N = 2; var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000, AutoCommitIntervalMs = 1000, EnableAutoCommit = false, EnablePartitionEof = true }; using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig) .SetRebalanceHandler((c, e) => { if (e.IsAssignment) { Assert.Single(e.Partitions); c.Assign(new TopicPartitionOffset(singlePartitionTopic, firstProduced.Partition, firstProduced.Offset)); } }) .Build()) { consumer.Subscribe(singlePartitionTopic); int msgCnt = 0; while (true) { var record = consumer.Consume(TimeSpan.FromMilliseconds(100)); if (record == null) { continue; } if (record.IsPartitionEOF) { break; } msgCnt += 1; } Assert.Equal(msgCnt, N); Thread.Sleep(TimeSpan.FromSeconds(3)); var committed = consumer.Committed(new [] { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); // if this was committing, would expect the committed offset to be first committed offset + N // (don't need to subtract 1 since the next message to be consumed is the value that is committed). Assert.NotEqual(firstProduced.Offset + N, committed[0].Offset); consumer.Close(); } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_AutoCommit"); }
static void Main(string[] args) { Console.WriteLine($"PatientService({instanceId}) - starting"); var conf = new ConsumerConfig { GroupId = consumer_group, BootstrapServers = kafkaBrokers, // Note: The AutoOffsetReset property determines the start offset in the event // there are not yet any committed offsets for the consumer group for the // topic/partitions of interest. By default, offsets are committed // automatically, so in this example, consumption will only start from the // earliest message in the topic 'my-topic' the first time you run the program. AutoOffsetReset = AutoOffsetReset.Earliest }; var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }; try { using (var consumer = new ConsumerBuilder <Ignore, string>(conf).Build()) { consumer.Subscribe(topic); try { while (true) { try { var response = consumer.Consume(); var rxEvent = JsonSerializer.Deserialize(response.Value, typeof(RxPrescribedEvent), jsonOptions) as RxPrescribedEvent; if (rxEvent.Medication == null) { Console.WriteLine("***Throwing away junk ***"); continue; } Console.WriteLine($"PatientService({instanceId}) - Medication {rxEvent.Medication.DrugName} prescribed notification sent to patient {rxEvent.Patient.LastName}, {rxEvent.Patient.FirstName}\n"); } catch (ConsumeException e) { Console.WriteLine($"PatientService({instanceId}) - Event consuming error occured: {e.Error.Reason}"); } catch (JsonException e) { Console.WriteLine($"PatientService({instanceId}) - JSON error occured: {e.Message}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. consumer.Close(); } } } catch (Exception ex) { Console.WriteLine($"PatientService({instanceId}) - unhandled exception: {ex}"); } }
public async Task KafkaConsumer() { List <string> topicListFromDB = chatsService.GetTopics(); ConsumerConfig config = JsonConvert.DeserializeObject <ConsumerConfig>(System.IO.File.ReadAllText(Constant.ConsumerConfigFilePath)); var topicListFromServer = AdminClient.RemoveAdminTopics(AdminClient.getMetadata(Constant.BrokerIP)); foreach (string topic in topicListFromServer) { if (!topicListFromDB.Contains(topic)) { Chat chat = new Chat(topic); chatsService.Create(chat); } } using (var consumer = new ConsumerBuilder <Ignore, byte[]>(config).Build()) { string fileName = ""; consumer.Subscribe(topicListFromServer); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { while (true) { try { var cr = consumer.Consume(cts.Token); Console.WriteLine(cr.Topic); // store the image if (cr.Topic.Equals("FileContent")) { if (fileName != "") { Console.WriteLine(cr.Value); FileStream myFile = System.IO.File.Open(@B3Image + fileName, FileMode.Create, FileAccess.Write); BinaryWriter myWriter = new BinaryWriter(myFile); myWriter.Write(cr.Value); myWriter.Close(); myFile.Close(); fileName = ""; } } else if (cr.Topic.Equals("FileName")) { fileName = Encoding.Default.GetString(cr.Value); } else { var incomming = new Chat(); try { ChatLine chatPackage = JsonConvert.DeserializeObject <List <ChatLine> >(Encoding.Default.GetString(cr.Value)).First(); incomming = new Chat(cr.Topic, chatPackage); } catch (Exception) { Console.WriteLine("Deserialize failed, check MongoDB if it is delivered from CLI."); ChatLine chatPackage = new ChatLine(Encoding.Default.GetString(cr.Value), "CLTestUser"); incomming = new Chat(cr.Topic, chatPackage); } if (topicListFromDB.Contains(cr.Topic)) { //Add new line to DB chatsService.UpdateByTopic(cr.Topic, incomming); } else { //create new Topic chatsService.Create(incomming); topicListFromDB.Add(cr.Topic); } var allChats = JsonConvert.SerializeObject ( chatsService.Get(), new JsonSerializerSettings { ContractResolver = new CamelCasePropertyNamesContractResolver() } ); await chatHub.SendMessage(allChats); } } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. consumer.Close(); } } }
public void Subscribe(Action <T> onchange, CacheNotifyAction cacheNotifyAction) { if (ClientConfig == null) { MemoryCacheNotify.Subscribe(onchange, cacheNotifyAction); return; } var channelName = GetChannelName(cacheNotifyAction); Cts[channelName] = new CancellationTokenSource(); Actions[channelName] = onchange; void action() { var conf = new ConsumerConfig(ClientConfig) { GroupId = Guid.NewGuid().ToString() }; using var c = new ConsumerBuilder <AscCacheItem, T>(conf) .SetErrorHandler((_, e) => Log.Error(e)) .SetKeyDeserializer(KeyDeserializer) .SetValueDeserializer(ValueDeserializer) .Build(); c.Assign(new TopicPartition(channelName, new Partition())); try { while (true) { try { var cr = c.Consume(Cts[channelName].Token); if (cr != null && cr.Value != null && !(new Guid(cr.Key.Id.ToByteArray())).Equals(Key) && Actions.TryGetValue(channelName, out var act)) { try { act(cr.Value); } catch (Exception e) { Log.Error("Kafka onmessage", e); } } } catch (ConsumeException e) { Log.Error(e); } } } catch (OperationCanceledException) { c.Close(); } } var task = new Task(action, TaskCreationOptions.LongRunning); task.Start(); }
static void Main(string[] args) { var config = new ConsumerConfig { GroupId = "re", BootstrapServers = "192.168.3.10:9092", AutoOffsetReset = AutoOffsetReset.Earliest }; using (var consumer = new ConsumerBuilder <Ignore, string>(config).Build()) { consumer.Subscribe("reserva"); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; try { while (true) { try { var cr = consumer.Consume(cts.Token); Console.WriteLine(cr.Value); OrdenCompraModel ordenCompra = JsonConvert.DeserializeObject <OrdenCompraModel>(cr.Value); ordenCompra.sEstado = "reservado"; ProductoDAO productoDAO = new ProductoDAO(); productoDAO.actualizarStock(ordenCompra.lDetalleCompra); OrdenCompraDAO ordenCompraDAO = new OrdenCompraDAO(); ordenCompraDAO.registrarOrdenCompra(ordenCompra); var config2 = new ProducerConfig { BootstrapServers = "192.168.3.10:9092" }; Action <DeliveryReport <Null, string> > handler = r => Console.WriteLine(!r.Error.IsError ? $"Delivered message to {r.TopicPartitionOffset}" : $"Delivery Error: {r.Error.Reason}"); using (var producer = new ProducerBuilder <Null, string>(config2).Build()) { producer.ProduceAsync("factura", new Message <Null, string> { Value = JsonConvert.SerializeObject(ordenCompra) }); producer.Flush(TimeSpan.FromSeconds(10)); } } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { consumer.Close(); } } }
static async Task Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { // Note: you can specify more than one schema registry url using the // schema.registry.url property for redundancy (comma separated list). // The property name is not plural to follow the convention set by // the Java implementation. Url = schemaRegistryUrl, }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = "protobuf-example-consumer-group" }; CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var consumer = new ConsumerBuilder <string, User>(consumerConfig) .SetValueDeserializer(new ProtobufDeserializer <User>().AsSyncOverAsync()) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { consumer.Subscribe(topicName); try { while (true) { try { var consumeResult = consumer.Consume(cts.Token); Console.WriteLine($"user name: {consumeResult.Message.Key}, favorite color: {consumeResult.Message.Value.FavoriteColor}"); } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { consumer.Close(); } } }); using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, User>(producerConfig) .SetValueSerializer(new ProtobufSerializer <User>(schemaRegistry)) .Build()) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit."); int i = 0; string text; while ((text = Console.ReadLine()) != "q") { User user = new User { Name = text, FavoriteColor = "green", FavoriteNumber = i++ }; await producer .ProduceAsync(topicName, new Message <string, User> { Key = text, Value = user }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); }
public void Consumer_Poll_Error(string bootstrapServers) { LogToFile("start Consumer_Poll_Error"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; TopicPartitionOffset firstProduced = null; using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build()) { var keyData = Encoding.UTF8.GetBytes("key"); firstProduced = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> { Key = keyData }).Result.TopicPartitionOffset; var valData = Encoding.UTF8.GetBytes("val"); producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> { Value = valData }); Assert.True(producer.Flush(TimeSpan.FromSeconds(10)) == 0); } var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000, EnablePartitionEof = true }; // test key deserialization error behavior using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig) .SetRebalanceHandler((c, e) => { if (e.IsAssignment) { Assert.Single(e.Partitions); Assert.Equal(firstProduced.TopicPartition, e.Partitions[0]); c.Assign(e.Partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset))); } else { c.Unassign(); } }) .Build()) { consumer.Subscribe(singlePartitionTopic); int msgCnt = 0; int errCnt = 0; while (true) { var s = consumer.Subscription; try { var record = consumer.Consume(TimeSpan.FromSeconds(10)); if (record == null) { continue; } if (record.IsPartitionEOF) { break; } msgCnt += 1; } catch (ConsumeException e) { errCnt += 1; Assert.Equal(ErrorCode.Local_KeyDeserialization, e.Error.Code); Assert.Equal(firstProduced.Offset.Value, e.ConsumerRecord.Offset.Value); } } Assert.Equal(1, msgCnt); Assert.Equal(1, errCnt); consumer.Close(); } // test value deserialization error behavior. using (var consumer = new ConsumerBuilder <string, Null>(consumerConfig) .SetRebalanceHandler((c, e) => { if (e.IsAssignment) { Assert.Single(e.Partitions); Assert.Equal(firstProduced.TopicPartition, e.Partitions[0]); c.Assign(e.Partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset))); } else { c.Unassign(); } }) .Build()) { consumer.Subscribe(singlePartitionTopic); int msgCnt = 0; int errCnt = 0; while (true) { try { var record = consumer.Consume(TimeSpan.FromSeconds(10)); if (record == null) { continue; } if (record.IsPartitionEOF) { break; } msgCnt += 1; } catch (ConsumeException e) { errCnt += 1; Assert.Equal(ErrorCode.Local_ValueDeserialization, e.Error.Code); Assert.Equal(firstProduced.Offset.Value + 1, e.ConsumerRecord.Offset.Value); } } Assert.Equal(1, msgCnt); Assert.Equal(1, errCnt); consumer.Close(); } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_Poll_Error"); }
private static void Consume() { var conf = new ConsumerConfig { GroupId = "test-consumer-group", BootstrapServers = "omnibus-01.srvs.cloudkafka.com:9094,omnibus-02.srvs.cloudkafka.com:9094,omnibus-03.srvs.cloudkafka.com:9094", SaslUsername = "******", SaslPassword = "******", SecurityProtocol = SecurityProtocol.SaslSsl, SaslMechanism = SaslMechanism.ScramSha256, AutoOffsetReset = AutoOffsetReset.Latest, EnableAutoCommit = true }; var counter = 0; using (var c = new ConsumerBuilder <Null, User>(conf) .SetValueDeserializer(new CustomDeserializer <User>()) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { c.Subscribe("ynm8ml8b-HelloWorld"); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { while (true) { var cr = new ConsumeResult <Null, User>(); try { cr = c.Consume(cts.Token); counter++; Console.WriteLine($"Counter:{counter}"); if (cr.Message != null) { for (int i = 0; i < cr.Message.Headers.Count; i++) { var key = cr.Message.Headers[i].Key; var value = Encoding.UTF8.GetString(cr.Message.Headers[i].GetValueBytes()); Console.WriteLine($"Chave:{key} || Valor:{value}"); } Console.WriteLine($"Consumed message id '{JsonConvert.SerializeObject(cr.Message.Value)}' at: '{cr.TopicPartitionOffset}'."); } else { cr.Message = new Message <Null, User>(); } } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. c.Close(); } } }
static async Task Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { // Note: you can specify more than one schema registry url using the // schema.registry.url property for redundancy (comma separated list). // The property name is not plural to follow the convention set by // the Java implementation. Url = schemaRegistryUrl }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = "json-example-consumer-group" }; // Note: Specifying json serializer configuration is optional. var jsonSerializerConfig = new JsonSerializerConfig { BufferBytes = 100 }; CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var consumer = new ConsumerBuilder <Null, Person>(consumerConfig) .SetValueDeserializer(new JsonDeserializer <Person>().AsSyncOverAsync()) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { consumer.Subscribe(topicName); try { while (true) { try { var cr = consumer.Consume(cts.Token); Console.WriteLine($"Name: {cr.Message.Value.FirstName} {cr.Message.Value.LastName}, age: {cr.Message.Value.Age}"); } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { consumer.Close(); } } }); using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <Null, Person>(producerConfig) .SetValueSerializer(new JsonSerializer <Person>(schemaRegistry, jsonSerializerConfig)) .Build()) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter first names, q to exit."); int i = 0; string text; while ((text = Console.ReadLine()) != "q") { Person person = new Person { FirstName = text, LastName = "lastname", Age = i++ % 150 }; await producer .ProduceAsync(topicName, new Message <Null, Person> { Value = person }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) { // Note: a subject name strategy was not configured, so the default "Topic" was used. var schema = await schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName)); Console.WriteLine("\nThe JSON schema corresponding to the written data:"); Console.WriteLine(schema.SchemaString); } }
static async Task Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { // Note: you can specify more than one schema registry url using the // schema.registry.url property for redundancy (comma separated list). // The property name is not plural to follow the convention set by // the Java implementation. SchemaRegistryUrl = schemaRegistryUrl, // optional schema registry client properties: SchemaRegistryRequestTimeoutMs = 5000, SchemaRegistryMaxCachedSchemas = 10 }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = "avro-specific-example-group" }; var avroSerializerConfig = new AvroSerializerConfig { // optional Avro serializer properties: BufferBytes = 100, AutoRegisterSchemas = true }; // Note: The User class in this project was generated using the Confluent fork of the avrogen.exe tool // (avaliable from: https://github.com/confluentinc/avro/tree/confluent-fork) which includes modifications // that prevent namespace clashes with user namespaces that include the identifier 'Avro'. AvroSerializer // and AvroDeserializer are also compatible with classes generated by the official avrogen.exe tool // (available from: https://github.com/apache/avro), with the above limitation. CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var consumer = new ConsumerBuilder <string, User>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry)) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { consumer.Subscribe(topicName); while (!cts.Token.IsCancellationRequested) { try { var consumeResult = consumer.Consume(cts.Token); Console.WriteLine($"user key name: {consumeResult.Message.Key}, user value favorite color: {consumeResult.Value.favorite_color}"); } catch (ConsumeException e) { Console.WriteLine("Consume error: " + e.Error.Reason); } } consumer.Close(); } }, cts.Token); using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, User>(producerConfig) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(new AvroSerializer <User>(schemaRegistry)) .Build()) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit."); int i = 0; string text; while ((text = Console.ReadLine()) != "q") { User user = new User { name = text, favorite_color = "green", favorite_number = i++ }; await producer .ProduceAsync(topicName, new Message <string, User> { Key = text, Value = user }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); }
private static void ProduceConsume(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers, }; var avroSerializerConfig = new AvroSerializerConfig { SubjectNameStrategy = nameStrategy }; var adminClientConfig = new AdminClientConfig { BootstrapServers = bootstrapServers }; string topic = Guid.NewGuid().ToString(); using (var adminClient = new AdminClientBuilder(adminClientConfig).Build()) { adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, ProduceConsumeUser>(producerConfig) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) // Test ValueSubjectNameStrategy here, // and KeySubjectNameStrategy in ProduceConsumeGeneric. .SetValueSerializer(new AvroSerializer <ProduceConsumeUser>(schemaRegistry, avroSerializerConfig)) .Build()) { for (int i = 0; i < 100; ++i) { var user = new ProduceConsumeUser { name = i.ToString(), favorite_number = i, favorite_color = "blue" }; producer .ProduceAsync(topic, new Message <string, ProduceConsumeUser> { Key = user.name, Value = user }) .Wait(); } Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var consumer = new ConsumerBuilder <string, ProduceConsumeUser>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync()) .SetValueDeserializer(new AvroDeserializer <ProduceConsumeUser>(schemaRegistry).AsSyncOverAsync()) .SetErrorHandler((_, e) => Assert.True(false, e.Reason)) .Build()) { consumer.Subscribe(topic); int i = 0; while (true) { var record = consumer.Consume(TimeSpan.FromMilliseconds(100)); if (record == null) { continue; } if (record.IsPartitionEOF) { break; } Assert.Equal(i.ToString(), record.Message.Key); Assert.Equal(i.ToString(), record.Message.Value.name); Assert.Equal(i, record.Message.Value.favorite_number); Assert.Equal("blue", record.Message.Value.favorite_color); i += 1; } Assert.Equal(100, i); consumer.Close(); } // Check that what's in schema registry is what's expected. using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) { var subjects = schemaRegistry.GetAllSubjectsAsync().Result; if (nameStrategy == SubjectNameStrategy.TopicRecord) { Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count()); Assert.Single(subjects.Where(s => s == $"{topic}-key")); Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}")); } if (nameStrategy == SubjectNameStrategy.Topic) { Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count()); Assert.Single(subjects.Where(s => s == $"{topic}-key")); Assert.Single(subjects.Where(s => s == $"{topic}-value")); } if (nameStrategy == SubjectNameStrategy.Record) { Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key. Assert.Single(subjects.Where(s => s == $"{topic}-key")); Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}")); } } }
/// <summary> /// Main method for console app. /// </summary> /// <param name="args">No arguments used.</param> public static void Main(string[] args) { Console.WriteLine("Starting .net consumer."); // Configure the group id, location of the bootstrap server, default deserializers, // Confluent interceptors var consumerConfig = new ConsumerConfig { BootstrapServers = "kafka:9092", GroupId = "csharp-consumer-prev", AutoOffsetReset = AutoOffsetReset.Earliest, PluginLibraryPaths = "monitoring-interceptor", }; using (var consumer = new ConsumerBuilder<string, string>(consumerConfig) .SetPartitionsAssignedHandler((c, partitions) => { // Calculate the time 5 minutes ago var timestamp = new Confluent.Kafka.Timestamp(DateTime.Now.AddMinutes(-5)); // Build a list of TopicPartitionTimestamp var timestamps = partitions.Select(tp => new TopicPartitionTimestamp(tp, timestamp)); // TODO: Request the offsets for the start timestamp var offsets = c.OffsetsForTimes(??? foreach (var offset in offsets) { // TODO: Print the new offset for each partition Console.WriteLine(??? } // Return the new partition offsets return offsets; }) .Build()) { // Subscribe to our topic consumer.Subscribe(KafkaTopic); CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { int recordCount = 0; while (true) { try { // Poll for available records var cr = consumer.Consume(cts.Token); Console.WriteLine($"{cr.Key},{cr.Value}"); recordCount++; // Exit processing after 100 records if (recordCount >= 100) { break; } } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Clean up when the application exits Console.WriteLine("Closing consumer."); consumer.Close(); } } }
static async Task Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; string groupName = "avro-generic-example-group"; // var s = (RecordSchema)RecordSchema.Parse(File.ReadAllText("my-schema.json")); var s = (RecordSchema)RecordSchema.Parse( @"{ ""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"", ""type"": ""record"", ""name"": ""User"", ""fields"": [ {""name"": ""name"", ""type"": ""string""}, {""name"": ""favorite_number"", ""type"": [""int"", ""null""]}, {""name"": ""favorite_color"", ""type"": [""string"", ""null""]} ] }" ); CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryUrl })) using (var consumer = new ConsumerBuilder <string, GenericRecord>(new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = groupName }) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry)) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { consumer.Subscribe(topicName); while (!cts.Token.IsCancellationRequested) { try { var consumeResult = consumer.Consume(cts.Token); Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}"); } catch (ConsumeException e) { Console.WriteLine("Consume error: " + e.Error.Reason); } } consumer.Close(); } }, cts.Token); using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryUrl })) using (var producer = new ProducerBuilder <string, GenericRecord>(new ProducerConfig { BootstrapServers = bootstrapServers }) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry)) .Build()) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit."); int i = 0; string text; while ((text = Console.ReadLine()) != "q") { var record = new GenericRecord(s); record.Add("name", text); record.Add("favorite_number", i++); record.Add("favorite_color", "blue"); await producer .ProduceAsync(topicName, new Message <string, GenericRecord> { Key = text, Value = record }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); }
public void Consumer_PartitionEOF(string bootstrapServers) { LogToFile("start Consumer_PartitionEOF"); int N = 2; var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, EnablePartitionEof = false }; // no eof, non generic consumer case. using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig) .SetPartitionsAssignedHandler((c, partitions) => { Assert.Single(partitions); Assert.Equal(firstProduced.TopicPartition, partitions[0]); return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset))); }) .Build()) { consumer.Subscribe(singlePartitionTopic); var cr1 = consumer.Consume(); Assert.NotNull(cr1.Message); Assert.False(cr1.IsPartitionEOF); var cr2 = consumer.Consume(); Assert.NotNull(cr2.Message); Assert.False(cr2.IsPartitionEOF); var cr3 = consumer.Consume(TimeSpan.FromSeconds(1)); Assert.Null(cr3); consumer.Close(); } // no eof, generic consumer case. using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig) .SetPartitionsAssignedHandler((c, partitions) => { Assert.Single(partitions); Assert.Equal(firstProduced.TopicPartition, partitions[0]); return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset))); }) .Build()) { consumer.Subscribe(singlePartitionTopic); var cr1 = consumer.Consume(); Assert.NotNull(cr1.Message); Assert.False(cr1.IsPartitionEOF); var cr2 = consumer.Consume(); Assert.NotNull(cr2.Message); Assert.False(cr2.IsPartitionEOF); var cr3 = consumer.Consume(TimeSpan.FromSeconds(1)); Assert.Null(cr3); consumer.Close(); } consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, EnablePartitionEof = true }; // eof, non-generic consumer case. using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig) .SetPartitionsAssignedHandler((c, partitions) => { Assert.Single(partitions); Assert.Equal(firstProduced.TopicPartition, partitions[0]); return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset))); }) .Build()) { consumer.Subscribe(singlePartitionTopic); var cr1 = consumer.Consume(); Assert.NotNull(cr1.Message); Assert.False(cr1.IsPartitionEOF); var cr2 = consumer.Consume(); Assert.NotNull(cr2.Message); Assert.False(cr2.IsPartitionEOF); var cr3 = consumer.Consume(); Assert.Null(cr3.Message); Assert.True(cr3.IsPartitionEOF); var cr4 = consumer.Consume(TimeSpan.FromSeconds(1)); Assert.Null(cr4); consumer.Close(); } // eof, generic consumer case. using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig) .SetPartitionsAssignedHandler((c, partitions) => { Assert.Single(partitions); Assert.Equal(firstProduced.TopicPartition, partitions[0]); return(partitions.Select(p => new TopicPartitionOffset(p, firstProduced.Offset))); }) .Build()) { consumer.Subscribe(singlePartitionTopic); var cr1 = consumer.Consume(); Assert.NotNull(cr1.Message); Assert.False(cr1.IsPartitionEOF); var cr2 = consumer.Consume(); Assert.NotNull(cr2.Message); Assert.False(cr2.IsPartitionEOF); var cr3 = consumer.Consume(); Assert.Null(cr3.Message); Assert.True(cr3.IsPartitionEOF); var cr4 = consumer.Consume(TimeSpan.FromSeconds(1)); Assert.Null(cr4); consumer.Close(); } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_PartitionEOF"); }
/// <summary> /// In this example /// - consumer group functionality (i.e. .Subscribe + offset commits) is not used. /// - the consumer is manually assigned to a partition and always starts consumption /// from a specific offset (0). /// </summary> public static void Run_ManualAssign(string brokerList, List <string> topics, CancellationToken cancellationToken) { var config = new ConsumerConfig { // the group.id property must be specified when creating a consumer, even // if you do not intend to use any consumer group functionality. GroupId = new Guid().ToString(), BootstrapServers = brokerList, // partition offsets can be committed to a group even by consumers not // subscribed to the group. in this example, auto commit is disabled // to prevent this from occurring. EnableAutoCommit = true }; using (var consumer = new ConsumerBuilder <string, string>(config) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { List <TopicMessages> messages = new List <TopicMessages>(); consumer.Assign(topics.Select(topic => new TopicPartitionOffset(topic, 0, Offset.Beginning)).ToList()); try { while (true) { try { var consumeResult = consumer.Consume(cancellationToken); // Note: End of partition notification has not been enabled, so // it is guaranteed that the ConsumeResult instance corresponds // to a Message, and not a PartitionEOF event. Console.WriteLine($"Received message at {consumeResult.TopicPartitionOffset}: ${consumeResult.Message.Value}"); messages.Add(new TopicMessages() { Topic = consumeResult.Topic, Offset = consumeResult.Offset.Value, Key = consumeResult.Message.Key, Value = consumeResult.Message.Value, }); //Console.WriteLine($"Received message at topic: {consumeResult.Topic}, partition: {consumeResult.Partition}, message key: {consumeResult.Message.Key}, offset: {consumeResult.Offset}, topic partition offset: {consumeResult.TopicPartitionOffset}"); } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { Console.WriteLine("Closing consumer."); consumer.Close(); } //Only save the results at the end //We can make that it is saved when we have a certain number of elements using (var db = new TestDbContext()) { db.Messages.AddRange( messages ); var count = db.SaveChanges(); Console.WriteLine("{0} records saved to database", count); } } }