/// <summary> /// Sets an Avro deserializer for keys. /// </summary> /// <typeparam name="TKey"> /// The type of key to be deserialized. /// </typeparam> /// <typeparam name="TValue"> /// The type of value to be deserialized. /// </typeparam> /// <param name="consumerBuilder"> /// A <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="deserializerBuilder"> /// A deserializer builder. /// </param> /// <param name="subject"> /// The subject of the schema that should be used to deserialize keys. /// </param> /// <param name="version"> /// The version of the subject to be resolved. /// </param> /// <returns> /// <paramref name="consumerBuilder" /> with an Avro deserializer configured for /// <typeparamref name="TKey" />. /// </returns> public static async Task <ConsumerBuilder <TKey, TValue> > SetAvroKeyDeserializer <TKey, TValue>( this ConsumerBuilder <TKey, TValue> consumerBuilder, ISchemaRegistryDeserializerBuilder deserializerBuilder, string subject, int version) => consumerBuilder.SetKeyDeserializer( await deserializerBuilder.Build <TKey>(subject, version).ConfigureAwait(false));
/// <summary> /// Set the message key deserializer. /// </summary> /// <param name="consumerBuilder"> /// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="registryClient"> /// The client to use for Schema Registry operations. The client should only be disposed /// after the consumer; the deserializer will use it to request schemas as messages are /// being consumed. /// </param> public static ConsumerBuilder <TKey, TValue> SetAvroKeyDeserializer <TKey, TValue>( this ConsumerBuilder <TKey, TValue> consumerBuilder, ISchemaRegistryClient registryClient ) => consumerBuilder.SetKeyDeserializer( new AsyncSchemaRegistryDeserializer <TKey>( registryClient ).AsSyncOverAsync());
/// <summary> /// Set the message key deserializer. /// </summary> /// <param name="consumerBuilder"> /// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="registryConfiguration"> /// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is /// highly recommended. /// </param> public static ConsumerBuilder <TKey, TValue> SetAvroKeyDeserializer <TKey, TValue>( this ConsumerBuilder <TKey, TValue> consumerBuilder, IEnumerable <KeyValuePair <string, string> > registryConfiguration ) => consumerBuilder.SetKeyDeserializer( new AsyncSchemaRegistryDeserializer <TKey>( registryConfiguration ).AsSyncOverAsync());
/// <summary> /// Need to return a <see cref="IConsumer{TKey, TValue}"/> for unit tests. /// Unfortunately <see cref="ConsumerBuilder{TKey, TValue}"/> returns <see cref="Consumer{TKey, TValue}"/> /// </summary> protected virtual IConsumer <TKey, TValue> CreateConsumer( ConsumerConfig config, Action <Consumer <TKey, TValue>, Error> errorHandler, Action <IConsumer <TKey, TValue>, List <TopicPartition> > partitionsAssignedHandler, Action <IConsumer <TKey, TValue>, List <TopicPartitionOffset> > partitionsRevokedHandler, IAsyncDeserializer <TValue> asyncValueDeserializer = null, IDeserializer <TValue> valueDeserializer = null, IAsyncDeserializer <TKey> keyDeserializer = null ) { var builder = new ConsumerBuilder <TKey, TValue>(config) .SetErrorHandler(errorHandler) .SetPartitionsAssignedHandler(partitionsAssignedHandler) .SetPartitionsRevokedHandler(partitionsRevokedHandler); if (keyDeserializer != null) { builder.SetKeyDeserializer(keyDeserializer); } if (asyncValueDeserializer != null) { builder.SetValueDeserializer(asyncValueDeserializer); } else if (valueDeserializer != null) { builder.SetValueDeserializer(valueDeserializer); } return(builder.Build()); }
#pragma warning disable 8618 public EventConsumer( IEventDeserializer eventDeserializer, EventConsumerConfig config, ILogger <EventConsumer <TAggregate, TAggregateId, TDeserializer> > logger) { _eventDeserializer = eventDeserializer; _logger = logger; var aggregateType = typeof(TAggregate); var consumerConfig = new ConsumerConfig { GroupId = config.ConsumerGroup, BootstrapServers = config.KafkaConnectionString, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; var consumerBuilder = new ConsumerBuilder <TAggregateId, string>(consumerConfig); var keyDeserializerFactory = new KeyDeserializerFactory(); consumerBuilder.SetKeyDeserializer(keyDeserializerFactory.Create <TDeserializer, TAggregateId>()); _eventConsumer = consumerBuilder.Build(); var topicName = $"{config.TopicBaseName}-{aggregateType.Name}"; _eventConsumer.Subscribe(topicName); }
public static IConsumer <T1, T2> Build <T1, T2>(this ConsumerBuilder <T1, T2> builder, JsonSerializerSettings serializerSettings) where T1 : class where T2 : class { return(builder .SetKeyDeserializer(new JsonNETKafkaSerializer <T1>(serializerSettings)) .SetValueDeserializer(new JsonNETKafkaSerializer <T2>(serializerSettings)) .Build()); }
public ConsumerConnectionBuilder <TKey, TValue> WithSchemaRegistry(string url) { if (typeof(TKey) == typeof(string) || typeof(TValue) == typeof(string)) { return(this); } var schemaRegistryConfig = new SchemaRegistryConfig() { SchemaRegistryUrl = url }; var schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig); var deserializerConfig = new AvroDeserializerConfig(); consumerBuilder.SetKeyDeserializer(new AvroDeserializer <TKey>(schemaRegistryClient, deserializerConfig).AsSyncOverAsync()); consumerBuilder.SetValueDeserializer(new AvroDeserializer <TValue>(schemaRegistryClient, deserializerConfig).AsSyncOverAsync()); return(this); }
public ConsumerBuilder <TKey, TValue> Configure <TKey, TValue>( ConsumerBuilder <TKey, TValue> builder, IDeserializer <TKey> keyDeserializer, IDeserializer <TValue> valueDeserializer) { builder .SetKeyDeserializer(keyDeserializer) .SetValueDeserializer(valueDeserializer) .SetLogHandler(LogHandler) .SetErrorHandler(ErrorHandler) .SetStatisticsHandler(StatisticsHandler) .SetPartitionsAssignedHandler(PartitionsAssignedHandler) .SetPartitionsRevokedHandler(PartitionsRevokedHandler); return(builder); }
private ConsumerBuilder <K, T> CreateKafkaConsumerBuilder() { var kafkaConsumerBuilder = new ConsumerBuilder <K, T>(consumerConfig); kafkaConsumerBuilder.SetKeyDeserializer(keyDeserializer); kafkaConsumerBuilder.SetValueDeserializer(valueDeserializer); kafkaConsumerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError { IsFatal = e.IsFatal, Reason = e.Reason })); kafkaConsumerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics)); if (partitionsAssignedHandle != null && CommitEnable) { throw new ArgumentException("The partition assigned handle can not been setted if " + "'CommitEnable' property was setted to true."); } else if (partitionsAssignedHandle != null) { kafkaConsumerBuilder.SetPartitionsAssignedHandler(partitionsAssignedHandle); } return(kafkaConsumerBuilder); }
/// <summary> /// Sets an Avro deserializer for keys. /// </summary> /// <typeparam name="TKey"> /// The type of key to be deserialized. /// </typeparam> /// <typeparam name="TValue"> /// The type of value to be deserialized. /// </typeparam> /// <param name="consumerBuilder"> /// A <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="deserializerBuilder"> /// A deserializer builder. /// </param> /// <param name="id"> /// The ID of the schema that should be used to deserialize keys. /// </param> /// <returns> /// <paramref name="consumerBuilder" /> with an Avro deserializer configured for /// <typeparamref name="TKey" />. /// </returns> public static async Task <ConsumerBuilder <TKey, TValue> > SetAvroKeyDeserializer <TKey, TValue>( this ConsumerBuilder <TKey, TValue> consumerBuilder, ISchemaRegistryDeserializerBuilder deserializerBuilder, int id) => consumerBuilder.SetKeyDeserializer( await deserializerBuilder.Build <TKey>(id).ConfigureAwait(false));
public async Task ConsumeMessages(CancellationToken stoppingToken) { var kafkaConfig = new ConsumerConfig { // Note: The AutoOffsetReset property determines the start offset in the event // there are not yet any committed offsets for the consumer group for the // topic/partitions of interest. By default, offsets are committed // automatically, so in this example, consumption will only start from the // earliest message in the topic 'my-topic' the first time you run the program. AutoOffsetReset = AutoOffsetReset.Earliest, GroupId = ConsumerGroupId, BootstrapServers = Kafka }; var consumer = new ConsumerBuilder <Key, Event>(kafkaConfig); consumer.SetErrorHandler(ConsumerErrorHandler); consumer.SetStatisticsHandler(ConsumerStatsHandler); using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig)) { consumer.SetKeyDeserializer(new AvroDeserializer <Key>(schemaRegistry).AsSyncOverAsync()); consumer.SetValueDeserializer(new AvroDeserializer <Event>(schemaRegistry).AsSyncOverAsync()); using (var c = consumer.Build()) { c.Subscribe(_topics.FirstOrDefault()); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. }; try { while (!stoppingToken.IsCancellationRequested) { try { var cr = c.Consume(stoppingToken); if (!cr.IsPartitionEOF) { ProcessMessage(cr); } } catch (ConsumeException e) { _logger.LogError($"Error occured in Kafka Consumer service: {e.Error.Reason + Environment.NewLine + e.StackTrace}"); } await Task.Delay(VerificationDelay, stoppingToken); } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. c.Close(); } } } }
/// <summary> /// Set the message key deserializer. /// </summary> /// <param name="consumerBuilder"> /// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="deserializerBuilder"> /// A deserializer builder. /// </param> /// <param name="subject"> /// The subject of the schema that should be used to deserialize keys. The latest version of /// the subject will be resolved. /// </param> public static async Task <ConsumerBuilder <TKey, TValue> > SetAvroKeyDeserializer <TKey, TValue>( this ConsumerBuilder <TKey, TValue> consumerBuilder, SchemaRegistryDeserializerBuilder deserializerBuilder, string subject ) => consumerBuilder.SetKeyDeserializer(await deserializerBuilder.Build <TKey>(subject));
public static void Main(string[] args) { var configuration = GetConfiguration(args); try { var prometheusConfig = configuration.GetSection("prometheusMetrics").Get <PrometheusConfig>(); MetricServer metricServer = null; if (prometheusConfig.Enabled) { metricServer = new MetricServer(port: prometheusConfig.Port); metricServer.Start(); } CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); var consumerConf = configuration.GetSection("consumerConf").Get <ConsumerConfig>(); consumerConf.GroupId = Guid.NewGuid().ToString(); ConsumerBuilder <Null, string> builder = new ConsumerBuilder <Null, string>(consumerConf); builder.SetErrorHandler((_, error) => { Console.WriteLine($"An error ocurred consuming the event: {error.Reason}"); if (error.IsFatal) { Environment.Exit(-1); } }); builder.HandleStatistics(new PrometheusConsumerStatisticsHandler(new string[] { "application" }, new string[] { "test-consumer-statistics" })); builder.SetKeyDeserializer(Deserializers.Null); builder.SetValueDeserializer(Deserializers.Utf8); using (var kafkaConsumer = builder.Build()) { kafkaConsumer.Subscribe(configuration.GetValue <string>("topic")); while (!cancellationTokenSource.IsCancellationRequested) { ConsumeResult <Null, string> consumedResult; try { consumedResult = kafkaConsumer.Consume(cancellationTokenSource.Token); if (null != consumedResult) { Console.WriteLine($"Received message: {consumedResult.Value}"); } } catch (Exception ex) { Console.WriteLine("An error occurred consuming the event.", ex); Environment.Exit(-2); } } } Console.WriteLine("Exit requested. Gracefully exiting..."); } catch (Exception ex) { Console.WriteLine("An error occurred while starting up the test.", ex); Environment.Exit(-2); } }
private static ConsumerBuilder <string, string> AddDeserializers(ConsumerBuilder <string, string> consumerBuilder) { return(consumerBuilder .SetKeyDeserializer(Serialization.StringDeserializer.Instance) .SetValueDeserializer(Serialization.StringDeserializer.Instance)); }
/// <summary> /// 初始化当前客户端 /// </summary> /// <param name="setting"></param> /// <returns></returns> public async Task Init(KafkaConsumerSetting setting) { var config = new ConsumerConfig { // 设置当前Kafka客户端分组, // 只要不更改group.id,每次重新消费kafka,都是从上次消费结束的地方继续开始 GroupId = setting.GroupId, BootstrapServers = setting.BootstrapServers, EnableAutoCommit = true, AutoOffsetReset = AutoOffsetReset.Earliest }; var consuming = true; var builder = new ConsumerBuilder <T1, T2>(config).SetErrorHandler((_, e) => { Logger.LogError($"Error: {e.Reason}"); consuming = !e.IsFatal; }); if (setting.ConsumeOffset != ConsumeOffset.Unspecified) { builder.SetPartitionsAssignedHandler((c, partitions) => { Logger.LogInformation($"Assigned partitions: [{string.Join(", ", partitions)}]"); return(partitions.Select(y => new TopicPartitionOffset(y, new Offset((long)setting.ConsumeOffset)))); }); } if (_keyDeserializer != null) { builder.SetKeyDeserializer(_keyDeserializer); } if (_valueDeserializer != null) { builder.SetValueDeserializer(_valueDeserializer); } using (_consumer = builder.Build()) { _consumer.Subscribe(setting.Topic); while (consuming && !_token.IsCancellationRequested) { try { var cr = _consumer.Consume(_token); try { await Consume(cr.Key, cr.Value, cr.Timestamp.UtcDateTime); } catch (Exception e) { Logger.LogError("KafkaConsumerManager.Consume error", e); } } catch (ConsumeException e) { Logger.LogError($"Error occured: {e.Error.Reason}"); } } } }
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken)) { funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding); CancellationTokenSource errorCts = new CancellationTokenSource(); CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token); CancellationToken compositeCancellationToken = compositeCts.Token; bool aMessageHasBeenProcessed = false; var cConfig = new ConsumerConfig { ClientId = $"{Name}-consumer-{instanceId}", GroupId = $"{Name}-group", BootstrapServers = BootstrapServers, EnableAutoCommit = true, EnableAutoOffsetStore = false, AutoOffsetReset = AutoOffsetReset.Latest }; if (DebugContext != null) { cConfig.Debug = DebugContext; } var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig); if (InKeyDeserializer != null) { cBuilder.SetKeyDeserializer(InKeyDeserializer); } if (InValueDeserializer != null) { cBuilder.SetValueDeserializer(InValueDeserializer); } if (Logger != null) { cBuilder.SetLogHandler((_, m) => { Logger(m); }); } cBuilder.SetErrorHandler((c, e) => { if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { // Logger.Log(e); errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var pConfig = new ProducerConfig { ClientId = $"{Name}-producer-{instanceId}", BootstrapServers = BootstrapServers, EnableIdempotence = true, LingerMs = 5, DeliveryReportFields = "none" }; if (DebugContext != null) { pConfig.Debug = DebugContext; } var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig); if (OutKeySerializer != null) { pBuilder.SetKeySerializer(OutKeySerializer); } if (OutValueSerializer != null) { pBuilder.SetValueSerializer(OutValueSerializer); } if (Logger != null) { pBuilder.SetLogHandler((_, m) => { Logger(m); }); } pBuilder.SetErrorHandler((p, e) => { if (e.IsFatal) { errorCts.Cancel(); return; } if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var partitionState = new Dictionary <TopicPartition, PartitionState>(); using (var producer = pBuilder.Build()) using (var consumer = cBuilder.Build()) { consumer.Subscribe(InputTopic); try { while (true) { ConsumeResult <TInKey, TInValue> cr; try { cr = consumer.Consume(compositeCancellationToken); } catch (ConsumeException ex) { if (ex.Error.Code == ErrorCode.Local_ValueDeserialization) { // For an in-depth discussion of what to do in the event of deserialization errors, refer to: // https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues if (ConsumeErrorTolerance == ErrorTolerance.All) { continue; } errorCts.Cancel(); // no error tolerance. } Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why? continue; } if (!partitionState.ContainsKey(cr.TopicPartition)) { partitionState.Add(cr.TopicPartition, new PartitionState(this)); } partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts); aMessageHasBeenProcessed = true; } } catch (OperationCanceledException) { } } if (errorCts.IsCancellationRequested) { throw new Exception("error occured, and we're failing fast."); } }