protected virtual void SetKeySerializer(ProducerBuilder <TKey, TValue> producerBuilder) { if (KeySerializer != null) { producerBuilder.SetKeySerializer(KeySerializer); } }
public KafkaEventPublisher( string topic, KafkaSettings settings, ISerializer <TKey> keySerializer, ISerializer <TEvent> valueSerializer, Func <TEvent, TKey> keyProvider) { _topic = topic; _keyProvider = keyProvider; var config = new ProducerConfig { BootstrapServers = string.Join(",", settings.BootstrapServers), Partitioner = Partitioner.Consistent }; var producerBuilder = new ProducerBuilder <TKey, TEvent>(config) .SetValueSerializer(valueSerializer); if (keySerializer != null) { producerBuilder.SetKeySerializer(keySerializer); } _producer = producerBuilder.Build(); }
/// <inheritdoc /> public ProducerBuilder <TKey, TValue> Configure <TKey, TValue>( ProducerBuilder <TKey, TValue> builder, ISerializer <TKey> keySerializer, ISerializer <TValue> valueSerializer) { if (builder == null) { throw new ArgumentNullException(nameof(builder)); } if (keySerializer == null) { throw new ArgumentNullException(nameof(keySerializer)); } if (valueSerializer == null) { throw new ArgumentNullException(nameof(valueSerializer)); } return(builder .SetKeySerializer(keySerializer) .SetValueSerializer(valueSerializer) .SetLogHandler(LogHandler) .SetErrorHandler(ErrorHandler) .SetStatisticsHandler(StatisticsHandler)); }
public static IProducer <T1, T2> Build <T1, T2>(this ProducerBuilder <T1, T2> builder, JsonSerializerSettings serializerSettings) where T1 : class where T2 : class { return(builder .SetKeySerializer(new JsonNETKafkaSerializer <T1>(serializerSettings)) .SetValueSerializer(new JsonNETKafkaSerializer <T2>(serializerSettings)) .Build()); }
public static IBusConfigurator UseKafkaTransport(this IBusConfigurator busConfigurator, KafkaConfiguration config, Action <IKafkaBusConfigurationBuilder> builderFunc) { busConfigurator.Services.AddSingleton(config); busConfigurator.Services.AddSingleton <IQueueReferenceFactory>(ctx => new QueueReferenceFactory(ctx, config.DefaultQueueReferenceCreator)); busConfigurator.Services.AddSingleton(ctx => { var kafkaConfig = ctx.GetRequiredService <KafkaConfiguration>(); return(new AdminClientConfig() { BootstrapServers = kafkaConfig.ConnectionString }); }); busConfigurator.Services.AddSingleton(ctx => { var adminClientConfig = ctx.GetRequiredService <AdminClientConfig>(); return(new AdminClientBuilder(adminClientConfig)); }); busConfigurator.Services.AddSingleton(ctx => { var kafkaConfig = ctx.GetRequiredService <KafkaConfiguration>(); return(new ProducerConfig() { BootstrapServers = kafkaConfig.ConnectionString }); }); busConfigurator.Services.AddSingleton(ctx => { var config = ctx.GetRequiredService <ProducerConfig>(); var builder = new ProducerBuilder <Guid, byte[]>(config); builder.SetKeySerializer(new KeySerializer <Guid>()); return(builder); }); busConfigurator.Services.AddSingleton <IProducer <Guid, byte[]> >(ctx => { var builder = ctx.GetRequiredService <ProducerBuilder <Guid, byte[]> >(); return(builder.Build()); }); busConfigurator.Services.AddTransient <IKafkaPublisherExecutor, KafkaPublisherExecutor>(); busConfigurator.Services.AddTransient <IPublisher, KafkaPublisher>(); busConfigurator.Services.AddTransient <IMessageParser, MessageParser>(); busConfigurator.Services.AddSingleton <IKafkaMessageHandler, KafkaMessageHandler>(); busConfigurator.Services.AddSingleton <IGroupIdFactory, DefaultGroupIdFactory>(); busConfigurator.Services.AddSingleton <IConsumerBuilderFactory, ConsumerBuilderFactory>(); builderFunc?.Invoke(new DefaultKafkaBusConfigurationBuilder(busConfigurator)); return(busConfigurator); }
public ProducerConnectionBuilder <TKey, TValue> WithSchemaRegistry(string url) { if (typeof(TKey) == typeof(string) || typeof(TValue) == typeof(string)) { return(this); } var schemaRegistryConfig = new SchemaRegistryConfig() { SchemaRegistryUrl = url }; var schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig); var serializerConfig = new AvroSerializerConfig(); producerBuilder.SetKeySerializer(new AvroSerializer <TKey>(schemaRegistryClient, serializerConfig)); producerBuilder.SetValueSerializer(new AvroSerializer <TValue>(schemaRegistryClient, serializerConfig)); return(this); }
/// <summary> /// Set the message key serializer. /// </summary> /// <param name="producerBuilder"> /// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="registryConfiguration"> /// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is /// highly recommended. /// </param> /// <param name="registerAutomatically"> /// Whether to automatically register schemas that match the type being serialized. /// </param> /// <param name="subjectNameBuilder"> /// A function that determines the subject name given the topic name and a component type /// (key or value). If none is provided, the default "{topic name}-{component}" naming /// convention will be used. /// </param> public static ProducerBuilder <TKey, TValue> SetAvroKeySerializer <TKey, TValue>( this ProducerBuilder <TKey, TValue> producerBuilder, IEnumerable <KeyValuePair <string, string> > registryConfiguration, bool registerAutomatically = false, Func <SerializationContext, string> subjectNameBuilder = null ) => producerBuilder.SetKeySerializer(new AsyncSchemaRegistrySerializer <TKey>( registryConfiguration, registerAutomatically: registerAutomatically, subjectNameBuilder: subjectNameBuilder ));
/// <summary> /// Set the message key serializer. /// </summary> /// <param name="producerBuilder"> /// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="registryClient"> /// A client to use for Schema Registry operations. The client should only be disposed /// after the producer; the serializer will use it to request schemas as messages are being /// produced. /// </param> /// <param name="registerAutomatically"> /// Whether to automatically register schemas that match the type being serialized. /// </param> /// <param name="subjectNameBuilder"> /// A function that determines the subject name given the topic name and a component type /// (key or value). If none is provided, the default "{topic name}-{component}" naming /// convention will be used. /// </param> public static ProducerBuilder <TKey, TValue> SetAvroKeySerializer <TKey, TValue>( this ProducerBuilder <TKey, TValue> producerBuilder, ISchemaRegistryClient registryClient, bool registerAutomatically = false, Func <SerializationContext, string> subjectNameBuilder = null ) => producerBuilder.SetKeySerializer(new AsyncSchemaRegistrySerializer <TKey>( registryClient, registerAutomatically: registerAutomatically, subjectNameBuilder: subjectNameBuilder ));
public KafkaSender(string url) { logger.Info("Create KafkaSender - " + url); ProducerConfig config = new ProducerConfig(); config.BootstrapServers = url; var builder = new ProducerBuilder <string, string>(config); builder.SetKeySerializer(Serializers.Utf8); producer = builder.Build(); }
public EventProducer(string topicBaseName, string kafkaConnString, ILogger <EventProducer <TA, TKey> > logger) { _logger = logger; var aggregateType = typeof(TA); _topicName = $"{topicBaseName}-{aggregateType.Name}"; var producerConfig = new ProducerConfig { BootstrapServers = kafkaConnString }; var producerBuilder = new ProducerBuilder <TKey, string>(producerConfig); producerBuilder.SetKeySerializer(new KeySerializer <TKey>()); _producer = producerBuilder.Build(); }
public KafkaProducer( ProducerConfig config, string avroSchema, ILogger logger) { this.logger = logger; var builder = new ProducerBuilder <TKey, TValue>(config); IAsyncSerializer <TValue> asyncValueSerializer = null; ISerializer <TValue> valueSerializer = null; IAsyncSerializer <TKey> keySerializer = null; if (!string.IsNullOrEmpty(avroSchema)) { var schemaRegistry = new LocalSchemaRegistry(avroSchema); asyncValueSerializer = new AvroSerializer <TValue>(schemaRegistry); } else { if (typeof(Google.Protobuf.IMessage).IsAssignableFrom(typeof(TValue))) { // protobuf: need to create using reflection due to generic requirements in ProtobufSerializer valueSerializer = (ISerializer <TValue>)Activator.CreateInstance(typeof(ProtobufSerializer <>).MakeGenericType(typeof(TValue))); } } if (asyncValueSerializer != null) { builder.SetValueSerializer(asyncValueSerializer); } else if (valueSerializer != null) { builder.SetValueSerializer(valueSerializer); } if (keySerializer != null) { builder.SetKeySerializer(keySerializer); } this.producer = builder.Build(); }
public static void Main(string[] args) { var configuration = GetConfiguration(args); try { var prometheusConfig = configuration.GetSection("prometheusMetrics").Get <PrometheusConfig>(); MetricServer metricServer = null; if (prometheusConfig.Enabled) { metricServer = new MetricServer(port: prometheusConfig.Port); metricServer.Start(); } CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); ProducerBuilder <Null, string> builder = new ProducerBuilder <Null, string>(configuration.GetSection("producerConf").Get <ProducerConfig>()); builder.SetErrorHandler((_, error) => { Console.WriteLine($"An error ocurred producing the event: {error.Reason}"); if (error.IsFatal) { Environment.Exit(-1); } }); builder.HandleStatistics(new PrometheusProducerStatisticsHandler(new string[] { "application" }, new string[] { "test-producer-statistics" })); builder.SetKeySerializer(Serializers.Null); builder.SetValueSerializer(Serializers.Utf8); using (var producer = builder.Build()) { Action <DeliveryReport <Null, string> > handler = r => { if (r.Error.IsError) { Console.WriteLine($"Delivery Error: {r.Error.Reason}"); } else { Console.WriteLine($"Delivered message to {r.TopicPartitionOffset}"); } }; int numMessages = 0; while (!cancellationTokenSource.IsCancellationRequested) { try { var dr = producer.ProduceAsync(configuration.GetValue <string>("topic"), new Message <Null, string> { Value = $"message {numMessages}" }); Console.WriteLine($"Delivered message {numMessages} : {dr.Result.Value}"); Thread.Sleep(1000); numMessages++; } catch (ProduceException <Null, string> e) { Console.WriteLine($"Delivery failed: {e.Error.Reason}"); } } Console.WriteLine("Exit requested."); producer.Flush(TimeSpan.FromSeconds(10)); } Console.WriteLine("Exit requested. Gracefully exiting..."); } catch (Exception ex) { Console.WriteLine("An error occurred while starting up the test.", ex); Environment.Exit(-2); } }
public string ProduceMessage(int keyId) { var message = $"Producer value-{keyId}"; var eventToSend = new Event { id = Guid.NewGuid().ToString(), date = DateTime.Now.ToString("u"), message = new Message { header = new Schemas.Header { msg_id = Guid.NewGuid().ToString(), msg_date = DateTime.Now.ToString("u") }, Body = message } }; try { var kafkaConfig = new ProducerConfig { BootstrapServers = Kafka }; var prod = new ProducerBuilder <Key, Event>(kafkaConfig); prod.SetErrorHandler(ProducerErrorHandler); prod.SetLogHandler(ProducerLogHandler); using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig)) { prod.SetKeySerializer(new AvroSerializer <Key>(schemaRegistry)); prod.SetValueSerializer(new AvroSerializer <Event>(schemaRegistry)); using (var p = prod.Build()) { try { var dr = p.ProduceAsync("logs", new Message <Key, Event> { Key = new Key { id = keyId }, Value = eventToSend }).GetAwaiter().GetResult(); _logger.LogInformation($"Kafka Producer service delivered the message '{message}' to Topic: {dr.Topic}"); } catch (ProduceException <Key, Event> e) { _logger.LogInformation($"Kafka Producer service Delivery failed: {e.Error.Reason}"); } } } } catch (OperationCanceledException oce) { _logger.LogError(oce, "OperationCanceled - Error in delivered message"); } catch (Exception e) { _logger.LogError(e, "Error in delivered message"); throw; } return($"The message '{message}' has been sent to the broker."); }
private void DumpMessages <M>(IEnumerable <M> messages, Func <M, Message <K, T> > getMessage, CancellationToken cancellationToken) { var producerBuilder = new ProducerBuilder <K, T>(producerConfig); producerBuilder.SetKeySerializer(keySerializer); producerBuilder.SetValueSerializer(valueSerializer); producerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError { IsFatal = e.IsFatal, Reason = e.Reason })); producerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics)); Stopwatch processTime = Stopwatch.StartNew(); using (var p = producerBuilder.Build()) { foreach (M message in messages) { if (cancellationToken.IsCancellationRequested) { break; } Policy.Handle <ProduceException <K, T> >() .WaitAndRetryForever(retryAttempt => TimeSpan.FromMilliseconds(Math.Min(100 * Math.Pow(2, retryAttempt), 10000)), (exception, timespan) => { var kafkaException = exception as ProduceException <K, T>; OnError?.Invoke(new StreamingError { IsFatal = kafkaException.Error.IsFatal, Reason = $"{kafkaException.Error.Reason}. The message with key {kafkaException.DeliveryResult.Key} in topic {kafkaException.DeliveryResult.Topic} will be resent on {timespan.TotalMilliseconds} ms." }); }) .Execute(() => { if (!cancellationToken.IsCancellationRequested) { p.Produce(this.topic, getMessage.Invoke(message), r => { if (r.Error.IsError) { OnError?.Invoke(new StreamingError { IsFatal = r.Error.IsFatal, Reason = r.Error.Reason }); } }); } }); if (processTime.ElapsedMilliseconds >= CommitTimeoutMs) { p.Flush(cancellationToken); if (!cancellationToken.IsCancellationRequested) { OnCommit?.Invoke(); processTime.Restart(); } } } p.Flush(cancellationToken); if (!cancellationToken.IsCancellationRequested) { OnCommit?.Invoke(); } } }
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken)) { funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding); CancellationTokenSource errorCts = new CancellationTokenSource(); CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token); CancellationToken compositeCancellationToken = compositeCts.Token; bool aMessageHasBeenProcessed = false; var cConfig = new ConsumerConfig { ClientId = $"{Name}-consumer-{instanceId}", GroupId = $"{Name}-group", BootstrapServers = BootstrapServers, EnableAutoCommit = true, EnableAutoOffsetStore = false, AutoOffsetReset = AutoOffsetReset.Latest }; if (DebugContext != null) { cConfig.Debug = DebugContext; } var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig); if (InKeyDeserializer != null) { cBuilder.SetKeyDeserializer(InKeyDeserializer); } if (InValueDeserializer != null) { cBuilder.SetValueDeserializer(InValueDeserializer); } if (Logger != null) { cBuilder.SetLogHandler((_, m) => { Logger(m); }); } cBuilder.SetErrorHandler((c, e) => { if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { // Logger.Log(e); errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var pConfig = new ProducerConfig { ClientId = $"{Name}-producer-{instanceId}", BootstrapServers = BootstrapServers, EnableIdempotence = true, LingerMs = 5, DeliveryReportFields = "none" }; if (DebugContext != null) { pConfig.Debug = DebugContext; } var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig); if (OutKeySerializer != null) { pBuilder.SetKeySerializer(OutKeySerializer); } if (OutValueSerializer != null) { pBuilder.SetValueSerializer(OutValueSerializer); } if (Logger != null) { pBuilder.SetLogHandler((_, m) => { Logger(m); }); } pBuilder.SetErrorHandler((p, e) => { if (e.IsFatal) { errorCts.Cancel(); return; } if (e.Code == ErrorCode.Local_AllBrokersDown || e.Code == ErrorCode.Local_Authentication) { if (!aMessageHasBeenProcessed) { errorCts.Cancel(); return; } } if (Logger != null) { Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason)); } }); var partitionState = new Dictionary <TopicPartition, PartitionState>(); using (var producer = pBuilder.Build()) using (var consumer = cBuilder.Build()) { consumer.Subscribe(InputTopic); try { while (true) { ConsumeResult <TInKey, TInValue> cr; try { cr = consumer.Consume(compositeCancellationToken); } catch (ConsumeException ex) { if (ex.Error.Code == ErrorCode.Local_ValueDeserialization) { // For an in-depth discussion of what to do in the event of deserialization errors, refer to: // https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues if (ConsumeErrorTolerance == ErrorTolerance.All) { continue; } errorCts.Cancel(); // no error tolerance. } Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why? continue; } if (!partitionState.ContainsKey(cr.TopicPartition)) { partitionState.Add(cr.TopicPartition, new PartitionState(this)); } partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts); aMessageHasBeenProcessed = true; } } catch (OperationCanceledException) { } } if (errorCts.IsCancellationRequested) { throw new Exception("error occured, and we're failing fast."); } }
/// <summary> /// Set the message key serializer. /// </summary> /// <param name="producerBuilder"> /// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="serializerBuilder"> /// A serializer builder. /// </param> /// <param name="id"> /// The ID of the schema that should be used to serialize keys. /// </param> public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>( this ProducerBuilder <TKey, TValue> producerBuilder, SchemaRegistrySerializerBuilder serializerBuilder, int id ) => producerBuilder.SetKeySerializer(await serializerBuilder.Build <TKey>(id));
/// <summary> /// Set the message key serializer. /// </summary> /// <param name="producerBuilder"> /// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="serializerBuilder"> /// A serializer builder. /// </param> /// <param name="subject"> /// The subject of the schema that should be used to serialize keys. /// </param> /// <param name="version"> /// The version of the subject to be resolved. /// </param> public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>( this ProducerBuilder <TKey, TValue> producerBuilder, SchemaRegistrySerializerBuilder serializerBuilder, string subject, int version ) => producerBuilder.SetKeySerializer(await serializerBuilder.Build <TKey>(subject, version));
/// <summary> /// Set the message key serializer. /// </summary> /// <param name="producerBuilder"> /// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured. /// </param> /// <param name="serializerBuilder"> /// A serializer builder. /// </param> /// <param name="subject"> /// The subject of the schema that should be used to serialize keys. The latest version of /// the subject will be resolved. /// </param> /// <param name="registerAutomatically"> /// Whether to automatically register a schema that matches <typeparamref name="TKey" /> /// if one does not already exist. /// </param> public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>( this ProducerBuilder <TKey, TValue> producerBuilder, SchemaRegistrySerializerBuilder serializerBuilder, string subject, bool registerAutomatically = false ) => producerBuilder.SetKeySerializer(await serializerBuilder.Build <TKey>(subject, registerAutomatically));