protected virtual void SetKeySerializer(ProducerBuilder <TKey, TValue> producerBuilder)
{
if (KeySerializer != null)
{
producerBuilder.SetKeySerializer(KeySerializer);
}
}
public KafkaEventPublisher(
string topic,
KafkaSettings settings,
ISerializer <TKey> keySerializer,
ISerializer <TEvent> valueSerializer,
Func <TEvent, TKey> keyProvider)
{
_topic = topic;
_keyProvider = keyProvider;
var config = new ProducerConfig
{
BootstrapServers = string.Join(",", settings.BootstrapServers),
Partitioner = Partitioner.Consistent
};
var producerBuilder = new ProducerBuilder <TKey, TEvent>(config)
.SetValueSerializer(valueSerializer);
if (keySerializer != null)
{
producerBuilder.SetKeySerializer(keySerializer);
}
_producer = producerBuilder.Build();
}
/// <inheritdoc />
public ProducerBuilder <TKey, TValue> Configure <TKey, TValue>(
ProducerBuilder <TKey, TValue> builder,
ISerializer <TKey> keySerializer,
ISerializer <TValue> valueSerializer)
{
if (builder == null)
{
throw new ArgumentNullException(nameof(builder));
}
if (keySerializer == null)
{
throw new ArgumentNullException(nameof(keySerializer));
}
if (valueSerializer == null)
{
throw new ArgumentNullException(nameof(valueSerializer));
}
return(builder
.SetKeySerializer(keySerializer)
.SetValueSerializer(valueSerializer)
.SetLogHandler(LogHandler)
.SetErrorHandler(ErrorHandler)
.SetStatisticsHandler(StatisticsHandler));
}
public static IProducer <T1, T2> Build <T1, T2>(this ProducerBuilder <T1, T2> builder, JsonSerializerSettings serializerSettings)
where T1 : class
where T2 : class
{
return(builder
.SetKeySerializer(new JsonNETKafkaSerializer <T1>(serializerSettings))
.SetValueSerializer(new JsonNETKafkaSerializer <T2>(serializerSettings))
.Build());
}
public static IBusConfigurator UseKafkaTransport(this IBusConfigurator busConfigurator,
KafkaConfiguration config,
Action <IKafkaBusConfigurationBuilder> builderFunc)
{
busConfigurator.Services.AddSingleton(config);
busConfigurator.Services.AddSingleton <IQueueReferenceFactory>(ctx => new QueueReferenceFactory(ctx, config.DefaultQueueReferenceCreator));
busConfigurator.Services.AddSingleton(ctx =>
{
var kafkaConfig = ctx.GetRequiredService <KafkaConfiguration>();
return(new AdminClientConfig()
{
BootstrapServers = kafkaConfig.ConnectionString
});
});
busConfigurator.Services.AddSingleton(ctx =>
{
var adminClientConfig = ctx.GetRequiredService <AdminClientConfig>();
return(new AdminClientBuilder(adminClientConfig));
});
busConfigurator.Services.AddSingleton(ctx =>
{
var kafkaConfig = ctx.GetRequiredService <KafkaConfiguration>();
return(new ProducerConfig()
{
BootstrapServers = kafkaConfig.ConnectionString
});
});
busConfigurator.Services.AddSingleton(ctx =>
{
var config = ctx.GetRequiredService <ProducerConfig>();
var builder = new ProducerBuilder <Guid, byte[]>(config);
builder.SetKeySerializer(new KeySerializer <Guid>());
return(builder);
});
busConfigurator.Services.AddSingleton <IProducer <Guid, byte[]> >(ctx =>
{
var builder = ctx.GetRequiredService <ProducerBuilder <Guid, byte[]> >();
return(builder.Build());
});
busConfigurator.Services.AddTransient <IKafkaPublisherExecutor, KafkaPublisherExecutor>();
busConfigurator.Services.AddTransient <IPublisher, KafkaPublisher>();
busConfigurator.Services.AddTransient <IMessageParser, MessageParser>();
busConfigurator.Services.AddSingleton <IKafkaMessageHandler, KafkaMessageHandler>();
busConfigurator.Services.AddSingleton <IGroupIdFactory, DefaultGroupIdFactory>();
busConfigurator.Services.AddSingleton <IConsumerBuilderFactory, ConsumerBuilderFactory>();
builderFunc?.Invoke(new DefaultKafkaBusConfigurationBuilder(busConfigurator));
return(busConfigurator);
}
public ProducerConnectionBuilder <TKey, TValue> WithSchemaRegistry(string url)
{
if (typeof(TKey) == typeof(string) || typeof(TValue) == typeof(string))
{
return(this);
}
var schemaRegistryConfig = new SchemaRegistryConfig()
{
SchemaRegistryUrl = url
};
var schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
var serializerConfig = new AvroSerializerConfig();
producerBuilder.SetKeySerializer(new AvroSerializer <TKey>(schemaRegistryClient, serializerConfig));
producerBuilder.SetValueSerializer(new AvroSerializer <TValue>(schemaRegistryClient, serializerConfig));
return(this);
}
/// <summary>
/// Set the message key serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryConfiguration">
/// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is
/// highly recommended.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
public static ProducerBuilder <TKey, TValue> SetAvroKeySerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
IEnumerable <KeyValuePair <string, string> > registryConfiguration,
bool registerAutomatically = false,
Func <SerializationContext, string> subjectNameBuilder = null
) => producerBuilder.SetKeySerializer(new AsyncSchemaRegistrySerializer <TKey>(
registryConfiguration,
registerAutomatically: registerAutomatically,
subjectNameBuilder: subjectNameBuilder
));
/// <summary>
/// Set the message key serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// A client to use for Schema Registry operations. The client should only be disposed
/// after the producer; the serializer will use it to request schemas as messages are being
/// produced.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
public static ProducerBuilder <TKey, TValue> SetAvroKeySerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
ISchemaRegistryClient registryClient,
bool registerAutomatically = false,
Func <SerializationContext, string> subjectNameBuilder = null
) => producerBuilder.SetKeySerializer(new AsyncSchemaRegistrySerializer <TKey>(
registryClient,
registerAutomatically: registerAutomatically,
subjectNameBuilder: subjectNameBuilder
));
public KafkaSender(string url)
{
logger.Info("Create KafkaSender - " + url);
ProducerConfig config = new ProducerConfig();
config.BootstrapServers = url;
var builder = new ProducerBuilder <string, string>(config);
builder.SetKeySerializer(Serializers.Utf8);
producer = builder.Build();
}
public EventProducer(string topicBaseName, string kafkaConnString, ILogger <EventProducer <TA, TKey> > logger)
{
_logger = logger;
var aggregateType = typeof(TA);
_topicName = $"{topicBaseName}-{aggregateType.Name}";
var producerConfig = new ProducerConfig {
BootstrapServers = kafkaConnString
};
var producerBuilder = new ProducerBuilder <TKey, string>(producerConfig);
producerBuilder.SetKeySerializer(new KeySerializer <TKey>());
_producer = producerBuilder.Build();
}
public KafkaProducer(
ProducerConfig config,
string avroSchema,
ILogger logger)
{
this.logger = logger;
var builder = new ProducerBuilder <TKey, TValue>(config);
IAsyncSerializer <TValue> asyncValueSerializer = null;
ISerializer <TValue> valueSerializer = null;
IAsyncSerializer <TKey> keySerializer = null;
if (!string.IsNullOrEmpty(avroSchema))
{
var schemaRegistry = new LocalSchemaRegistry(avroSchema);
asyncValueSerializer = new AvroSerializer <TValue>(schemaRegistry);
}
else
{
if (typeof(Google.Protobuf.IMessage).IsAssignableFrom(typeof(TValue)))
{
// protobuf: need to create using reflection due to generic requirements in ProtobufSerializer
valueSerializer = (ISerializer <TValue>)Activator.CreateInstance(typeof(ProtobufSerializer <>).MakeGenericType(typeof(TValue)));
}
}
if (asyncValueSerializer != null)
{
builder.SetValueSerializer(asyncValueSerializer);
}
else if (valueSerializer != null)
{
builder.SetValueSerializer(valueSerializer);
}
if (keySerializer != null)
{
builder.SetKeySerializer(keySerializer);
}
this.producer = builder.Build();
}
public static void Main(string[] args)
{
var configuration = GetConfiguration(args);
try
{
var prometheusConfig = configuration.GetSection("prometheusMetrics").Get <PrometheusConfig>();
MetricServer metricServer = null;
if (prometheusConfig.Enabled)
{
metricServer = new MetricServer(port: prometheusConfig.Port);
metricServer.Start();
}
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
ProducerBuilder <Null, string> builder = new ProducerBuilder <Null, string>(configuration.GetSection("producerConf").Get <ProducerConfig>());
builder.SetErrorHandler((_, error) =>
{
Console.WriteLine($"An error ocurred producing the event: {error.Reason}");
if (error.IsFatal)
{
Environment.Exit(-1);
}
});
builder.HandleStatistics(new PrometheusProducerStatisticsHandler(new string[] { "application" }, new string[] { "test-producer-statistics" }));
builder.SetKeySerializer(Serializers.Null);
builder.SetValueSerializer(Serializers.Utf8);
using (var producer = builder.Build())
{
Action <DeliveryReport <Null, string> > handler = r =>
{
if (r.Error.IsError)
{
Console.WriteLine($"Delivery Error: {r.Error.Reason}");
}
else
{
Console.WriteLine($"Delivered message to {r.TopicPartitionOffset}");
}
};
int numMessages = 0;
while (!cancellationTokenSource.IsCancellationRequested)
{
try
{
var dr = producer.ProduceAsync(configuration.GetValue <string>("topic"), new Message <Null, string> {
Value = $"message {numMessages}"
});
Console.WriteLine($"Delivered message {numMessages} : {dr.Result.Value}");
Thread.Sleep(1000);
numMessages++;
}
catch (ProduceException <Null, string> e)
{
Console.WriteLine($"Delivery failed: {e.Error.Reason}");
}
}
Console.WriteLine("Exit requested.");
producer.Flush(TimeSpan.FromSeconds(10));
}
Console.WriteLine("Exit requested. Gracefully exiting...");
}
catch (Exception ex)
{
Console.WriteLine("An error occurred while starting up the test.", ex);
Environment.Exit(-2);
}
}
public string ProduceMessage(int keyId)
{
var message = $"Producer value-{keyId}";
var eventToSend = new Event
{
id = Guid.NewGuid().ToString(),
date = DateTime.Now.ToString("u"),
message = new Message
{
header = new Schemas.Header
{
msg_id = Guid.NewGuid().ToString(),
msg_date = DateTime.Now.ToString("u")
},
Body = message
}
};
try
{
var kafkaConfig = new ProducerConfig {
BootstrapServers = Kafka
};
var prod = new ProducerBuilder <Key, Event>(kafkaConfig);
prod.SetErrorHandler(ProducerErrorHandler);
prod.SetLogHandler(ProducerLogHandler);
using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig))
{
prod.SetKeySerializer(new AvroSerializer <Key>(schemaRegistry));
prod.SetValueSerializer(new AvroSerializer <Event>(schemaRegistry));
using (var p = prod.Build())
{
try
{
var dr = p.ProduceAsync("logs", new Message <Key, Event> {
Key = new Key {
id = keyId
}, Value = eventToSend
}).GetAwaiter().GetResult();
_logger.LogInformation($"Kafka Producer service delivered the message '{message}' to Topic: {dr.Topic}");
}
catch (ProduceException <Key, Event> e)
{
_logger.LogInformation($"Kafka Producer service Delivery failed: {e.Error.Reason}");
}
}
}
}
catch (OperationCanceledException oce)
{
_logger.LogError(oce, "OperationCanceled - Error in delivered message");
}
catch (Exception e)
{
_logger.LogError(e, "Error in delivered message");
throw;
}
return($"The message '{message}' has been sent to the broker.");
}
private void DumpMessages <M>(IEnumerable <M> messages, Func <M, Message <K, T> > getMessage, CancellationToken cancellationToken)
{
var producerBuilder = new ProducerBuilder <K, T>(producerConfig);
producerBuilder.SetKeySerializer(keySerializer);
producerBuilder.SetValueSerializer(valueSerializer);
producerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError {
IsFatal = e.IsFatal, Reason = e.Reason
}));
producerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics));
Stopwatch processTime = Stopwatch.StartNew();
using (var p = producerBuilder.Build())
{
foreach (M message in messages)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
Policy.Handle <ProduceException <K, T> >()
.WaitAndRetryForever(retryAttempt => TimeSpan.FromMilliseconds(Math.Min(100 * Math.Pow(2, retryAttempt), 10000)),
(exception, timespan) =>
{
var kafkaException = exception as ProduceException <K, T>;
OnError?.Invoke(new StreamingError
{
IsFatal = kafkaException.Error.IsFatal,
Reason = $"{kafkaException.Error.Reason}. The message with key {kafkaException.DeliveryResult.Key} in topic {kafkaException.DeliveryResult.Topic} will be resent on {timespan.TotalMilliseconds} ms."
});
})
.Execute(() =>
{
if (!cancellationToken.IsCancellationRequested)
{
p.Produce(this.topic, getMessage.Invoke(message),
r =>
{
if (r.Error.IsError)
{
OnError?.Invoke(new StreamingError {
IsFatal = r.Error.IsFatal, Reason = r.Error.Reason
});
}
});
}
});
if (processTime.ElapsedMilliseconds >= CommitTimeoutMs)
{
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
processTime.Restart();
}
}
}
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
}
}
}
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken))
{
funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding);
CancellationTokenSource errorCts = new CancellationTokenSource();
CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token);
CancellationToken compositeCancellationToken = compositeCts.Token;
bool aMessageHasBeenProcessed = false;
var cConfig = new ConsumerConfig
{
ClientId = $"{Name}-consumer-{instanceId}",
GroupId = $"{Name}-group",
BootstrapServers = BootstrapServers,
EnableAutoCommit = true,
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Latest
};
if (DebugContext != null)
{
cConfig.Debug = DebugContext;
}
var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig);
if (InKeyDeserializer != null)
{
cBuilder.SetKeyDeserializer(InKeyDeserializer);
}
if (InValueDeserializer != null)
{
cBuilder.SetValueDeserializer(InValueDeserializer);
}
if (Logger != null)
{
cBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
cBuilder.SetErrorHandler((c, e) =>
{
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
// Logger.Log(e);
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var pConfig = new ProducerConfig
{
ClientId = $"{Name}-producer-{instanceId}",
BootstrapServers = BootstrapServers,
EnableIdempotence = true,
LingerMs = 5,
DeliveryReportFields = "none"
};
if (DebugContext != null)
{
pConfig.Debug = DebugContext;
}
var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig);
if (OutKeySerializer != null)
{
pBuilder.SetKeySerializer(OutKeySerializer);
}
if (OutValueSerializer != null)
{
pBuilder.SetValueSerializer(OutValueSerializer);
}
if (Logger != null)
{
pBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
pBuilder.SetErrorHandler((p, e) =>
{
if (e.IsFatal)
{
errorCts.Cancel();
return;
}
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var partitionState = new Dictionary <TopicPartition, PartitionState>();
using (var producer = pBuilder.Build())
using (var consumer = cBuilder.Build())
{
consumer.Subscribe(InputTopic);
try
{
while (true)
{
ConsumeResult <TInKey, TInValue> cr;
try
{
cr = consumer.Consume(compositeCancellationToken);
}
catch (ConsumeException ex)
{
if (ex.Error.Code == ErrorCode.Local_ValueDeserialization)
{
// For an in-depth discussion of what to do in the event of deserialization errors, refer to:
// https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues
if (ConsumeErrorTolerance == ErrorTolerance.All)
{
continue;
}
errorCts.Cancel(); // no error tolerance.
}
Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why?
continue;
}
if (!partitionState.ContainsKey(cr.TopicPartition))
{
partitionState.Add(cr.TopicPartition, new PartitionState(this));
}
partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts);
aMessageHasBeenProcessed = true;
}
}
catch (OperationCanceledException) { }
}
if (errorCts.IsCancellationRequested)
{
throw new Exception("error occured, and we're failing fast.");
}
}
/// <summary>
/// Set the message key serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="serializerBuilder">
/// A serializer builder.
/// </param>
/// <param name="id">
/// The ID of the schema that should be used to serialize keys.
/// </param>
public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
SchemaRegistrySerializerBuilder serializerBuilder,
int id
) => producerBuilder.SetKeySerializer(await serializerBuilder.Build <TKey>(id));
/// <summary>
/// Set the message key serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="serializerBuilder">
/// A serializer builder.
/// </param>
/// <param name="subject">
/// The subject of the schema that should be used to serialize keys.
/// </param>
/// <param name="version">
/// The version of the subject to be resolved.
/// </param>
public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
SchemaRegistrySerializerBuilder serializerBuilder,
string subject,
int version
) => producerBuilder.SetKeySerializer(await serializerBuilder.Build <TKey>(subject, version));
/// <summary>
/// Set the message key serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="serializerBuilder">
/// A serializer builder.
/// </param>
/// <param name="subject">
/// The subject of the schema that should be used to serialize keys. The latest version of
/// the subject will be resolved.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register a schema that matches <typeparamref name="TKey" />
/// if one does not already exist.
/// </param>
public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
SchemaRegistrySerializerBuilder serializerBuilder,
string subject,
bool registerAutomatically = false
) => producerBuilder.SetKeySerializer(await serializerBuilder.Build <TKey>(subject, registerAutomatically));