/// <summary>
///
/// </summary>
/// <param name="ProducerBuilder<K"></param>
/// <param name="builder"></param>
/// <param name="handler"></param>
/// <typeparam name="K"></typeparam>
/// <typeparam name="V"></typeparam>
/// <returns></returns>
public static void HandleStatistics <K, V>(this ProducerBuilder <K, V> builder, IStatisticsHandler handler)
{
builder.SetStatisticsHandler((_, json) =>
{
var statistics = JsonConvert.DeserializeObject <Statistics>(json);
handler.Publish(statistics);
});
}
private static ProducerBuilder <T, TV> GetProducerBuilderInstance <T, TV>(ProducerConfig configuration) where T : class where TV : class
{
var producer = new ProducerBuilder <T, TV>(configuration);
producer.SetErrorHandler((_, e) => Devon4NetLogger.Error(new ConsumerException($"Error code {e.Code} : {e.Reason}")));
producer.SetStatisticsHandler((_, json) => Devon4NetLogger.Information($"Statistics: {json}"));
producer.SetLogHandler((c, partitions) => { Devon4NetLogger.Information($"Kafka log handler: [{string.Join(", ", partitions)}]"); });
return(producer);
}
/// <inheritdoc cref="IConfluentProducerBuilder.Build" />
public IProducer <byte[]?, byte[]?> Build()
{
if (_config == null)
{
throw new InvalidOperationException("SetConfig must be called to provide the producer configuration.");
}
var builder = new ProducerBuilder <byte[]?, byte[]?>(_config);
if (_statisticsHandler != null)
{
builder.SetStatisticsHandler(_statisticsHandler);
}
return(builder.Build());
}
/// <summary>
/// producer构造器
/// </summary>
/// <returns></returns>
private ProducerBuilder <string, object> CreateProducerBuilder()
{
if (builder == null)
{
lock (this)
{
if (builder == null)
{
ProducerConfig config = new ProducerConfig();
config.BootstrapServers = BootstrapServers;
if (!string.IsNullOrEmpty(SaslUsername))
{
config.SaslUsername = SaslUsername;
config.SaslPassword = SaslPassword;
config.SaslMechanism = SaslMechanism.Plain;
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
}
//List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>();
//config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers));
//if (!string.IsNullOrEmpty(SaslUsername))
//{
// config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT"));
// config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN"));
// config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername));
// config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword));
//}
builder = new ProducerBuilder <string, object>(config);
Action <Delegate, object> tryCatchWrap = (@delegate, arg) =>
{
try
{
@delegate?.DynamicInvoke(arg);
}
catch { }
};
builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason)));
builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e));
builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e)));
builder.SetValueSerializer(new KafkaConverter());
}
}
}
return(builder);
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaSender"/> class.
/// </summary>
/// <param name="name">The name of the sender.</param>
/// <param name="topic">The topic to produce messages to.</param>
/// <param name="bootstrapServers">List of brokers as a CSV list of broker host or host:port.</param>
/// <param name="messageTimeoutMs">
/// Local message timeout. This value is only enforced locally and limits the time
/// a produced message waits for successful delivery. A time of 0 is infinite. This
/// is the maximum time librdkafka may use to deliver a message (including retries).
/// Delivery error occurs when either the retry count or the message timeout are
/// exceeded.
/// </param>
/// /// <param name="statisticsIntervalMs">
/// The statistics emit interval in milliseconds. Granularity is 1,000ms. An event handler must be attached to the
/// <see cref="StatisticsEmitted"/> event to receive the statistics data. Setting to 0 disables statistics.
/// </param>
public KafkaSender(string name, string topic, string bootstrapServers, int messageTimeoutMs = Constants.DefaultTimeout,
int statisticsIntervalMs = 0)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers));
MessageTimeoutMs = messageTimeoutMs;
var config = GetProducerConfig(bootstrapServers, messageTimeoutMs, statisticsIntervalMs);
var producerBuilder = new ProducerBuilder <string, byte[]>(config);
producerBuilder.SetErrorHandler(OnError);
producerBuilder.SetStatisticsHandler(OnStatisticsEmitted);
_producer = new Lazy <IProducer <string, byte[]> >(() => producerBuilder.Build());
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaSender"/> class.
/// </summary>
/// <param name="name">The name of the sender.</param>
/// <param name="topic">The topic to produce messages to.</param>
/// <param name="producerConfig">The configuration used in creation of the Kafka producer.</param>
public KafkaSender(string name, string topic, ProducerConfig producerConfig)
{
if (producerConfig is null)
{
throw new ArgumentNullException(nameof(producerConfig));
}
Name = name ?? throw new ArgumentNullException(nameof(name));
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
BootstrapServers = producerConfig.BootstrapServers;
MessageTimeoutMs = producerConfig.MessageTimeoutMs;
var producerBuilder = new ProducerBuilder <string, byte[]>(producerConfig);
producerBuilder.SetErrorHandler(OnError);
producerBuilder.SetStatisticsHandler(OnStatisticsEmitted);
_producer = new Lazy <IProducer <string, byte[]> >(() => producerBuilder.Build());
}
public IProducer <byte[], byte[]> GetProducer(ProducerConfig config)
{
ProducerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetProducerBuilder(config);
builder.SetLogHandler(loggerAdapter.LogProduce);
builder.SetErrorHandler(loggerAdapter.ErrorProduce);
if (exposeLibrdKafka)
{
// TODO : test librdkafka statistics with IntegrationTest (WIP see #82)
var producerStatisticsHandler = new ProducerStatisticsHandler(
config.ClientId,
streamConfig.ApplicationId,
(config as StreamizProducerConfig)?.ThreadId,
(config as StreamizProducerConfig)?.Id?.ToString());
producerStatisticsHandler.Register(MetricsRegistry);
builder.SetStatisticsHandler((c, stat) =>
{
var statistics = JsonConvert.DeserializeObject <Statistics>(stat);
producerStatisticsHandler.Publish(statistics);
});
}
return(builder.Build());
}
private void DumpMessages <M>(IEnumerable <M> messages, Func <M, Message <K, T> > getMessage, CancellationToken cancellationToken)
{
var producerBuilder = new ProducerBuilder <K, T>(producerConfig);
producerBuilder.SetKeySerializer(keySerializer);
producerBuilder.SetValueSerializer(valueSerializer);
producerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError {
IsFatal = e.IsFatal, Reason = e.Reason
}));
producerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics));
Stopwatch processTime = Stopwatch.StartNew();
using (var p = producerBuilder.Build())
{
foreach (M message in messages)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
Policy.Handle <ProduceException <K, T> >()
.WaitAndRetryForever(retryAttempt => TimeSpan.FromMilliseconds(Math.Min(100 * Math.Pow(2, retryAttempt), 10000)),
(exception, timespan) =>
{
var kafkaException = exception as ProduceException <K, T>;
OnError?.Invoke(new StreamingError
{
IsFatal = kafkaException.Error.IsFatal,
Reason = $"{kafkaException.Error.Reason}. The message with key {kafkaException.DeliveryResult.Key} in topic {kafkaException.DeliveryResult.Topic} will be resent on {timespan.TotalMilliseconds} ms."
});
})
.Execute(() =>
{
if (!cancellationToken.IsCancellationRequested)
{
p.Produce(this.topic, getMessage.Invoke(message),
r =>
{
if (r.Error.IsError)
{
OnError?.Invoke(new StreamingError {
IsFatal = r.Error.IsFatal, Reason = r.Error.Reason
});
}
});
}
});
if (processTime.ElapsedMilliseconds >= CommitTimeoutMs)
{
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
processTime.Restart();
}
}
}
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
}
}
}