public KafkaOutput(KafkaLoggingOptions options) : base(options.BufferedOutputOptions)
{
if (string.IsNullOrWhiteSpace(options.Topic))
{
throw new ArgumentException("Must not empty", $"{nameof(options)}.{nameof(options.Topic)}");
}
if (options.Serializer == null)
{
throw new ArgumentException("Must not null", $"{nameof(options)}.{nameof(options.Serializer)}");
}
_options = options;
if (!string.IsNullOrWhiteSpace(_options.ContentType))
{
_headers.Add("Content-Type", Encoding.UTF8.GetBytes(_options.ContentType));
}
var pb = new ProducerBuilder <Guid, IReadOnlyDictionary <string, object?> >(_options.ProducerConfig)
.SetKeySerializer(new GuiSerializer())
.SetValueSerializer(new ObjectSerializer(_options.Serializer));
if (_options.KafkaErrorHandler != null)
{
var handler = _options.KafkaErrorHandler;
pb.SetErrorHandler((_, error) => handler(error));
}
_producer = pb.Build();
}
public IProducer <Null, byte[]> CreateProducer(
KafkaQueueConfiguration config,
Action <IProducer <Null, byte[]>, LogMessage> logHandler = null,
Action <IProducer <Null, byte[]>, Error> errorHandler = null)
{
config.ThrowIfNull(nameof(config));
var builder = new ProducerBuilder <Null, byte[]>(new ProducerConfig
{
BootstrapServers = $"{config.Server}",
SaslKerberosKeytab = config.KeyTab,
SaslKerberosPrincipal = config.User,
SaslKerberosServiceName = config.ServiceName,
SecurityProtocol = config.Protocol,
SaslMechanism = config.Mechanism,
Debug = config.Debug
});
if (logHandler != null)
{
builder.SetLogHandler(logHandler);
}
if (errorHandler != null)
{
builder.SetErrorHandler(errorHandler);
}
return(builder.Build());
}
private void Init(ProducerConfig config)
{
if (_logger.IsDebug)
{
_logger.Debug($"Initializing {Name} type producer for Kafka...");
}
try
{
CachedSchemaRegistryClient schemaRegistry = new CachedSchemaRegistryClient(new[]
{
new KeyValuePair <string, string>(SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl, _schemaRegistryUrl)
});
var blockAvroSerializer = new AvroSerializer <Block>(schemaRegistry).AsSyncOverAsync();
var txAvroSerializer = new AvroSerializer <FullTransaction>(schemaRegistry).AsSyncOverAsync();
ProducerBuilder <Null, Block> blockProducerBuilder = new ProducerBuilder <Null, Block>(config);
blockProducerBuilder.SetValueSerializer(blockAvroSerializer);
blockProducerBuilder.SetErrorHandler((s, e) => _logger.Error(e.ToString()));
ProducerBuilder <Null, FullTransaction> txProducerBuilder = new ProducerBuilder <Null, FullTransaction>(config);
txProducerBuilder.SetValueSerializer(txAvroSerializer);
txProducerBuilder.SetErrorHandler((s, e) => _logger.Error(e.ToString()));
_producerBlocks = blockProducerBuilder.Build();
_producerTransactions = txProducerBuilder.Build();
_initialized = true;
if (_logger.IsDebug)
{
_logger.Debug($"Initialized {Name} type producer for Kafka.");
}
}
catch (Exception e)
{
_logger.Error(e.Message, e);
}
}
public static void Producer()
{
var config = new ProducerConfig {
BootstrapServers = mBootstrapServers
};
Action <DeliveryReport <Null, string> > handler = r =>
Console.WriteLine(!r.Error.IsError
? $"Delivered message to {r.TopicPartitionOffset}"
: $"Delivery Error: {r.Error.Reason}");
var producerBuilder = new ProducerBuilder <Null, string>(config);
// 错误日志监视
producerBuilder.SetErrorHandler((p, msg) =>
{
Console.WriteLine($"Producer_Erro信息：Code：{msg.Code}；Reason：{msg.Reason}；IsError：{msg.IsError}");
});
using (var producer = producerBuilder.Build())
{
for (int i = 0; i < 5; i++)
{
// 异步发送消息到主题
producer.Produce(mTopick, new Message <Null, string> {
Value = i.ToString()
}, handler);
}
// 3后 Flush到磁盘
producer.Flush(TimeSpan.FromSeconds(3));
}
}
public IProducer <string, string> Create()
{
var config = new ProducerConfig(_configuration.GetProducerConfiguration());
var builder = new ProducerBuilder <string, string>(config);
builder.SetErrorHandler(OnKafkaError);
return(builder.Build());
}
public IProducer <byte[], byte[]> GetProducer(ProducerConfig config)
{
ProducerBuilder <byte[], byte[]> builder = new ProducerBuilder <byte[], byte[]>(config);
builder.SetLogHandler(loggerAdapter.LogProduce);
builder.SetErrorHandler(loggerAdapter.ErrorProduce);
return(builder.Build());
}
public KafkaProducer(IKafkaConfig config, K serializer)
{
_config = config;
_serializer = serializer;
_producerBldr = new ProducerBuilder <Null, string>(_config.ProducerConfig());
_producerBldr.SetErrorHandler(ErrorHandler);
_producer = _producerBldr.Build();
}
private static ProducerBuilder <T, TV> GetProducerBuilderInstance <T, TV>(ProducerConfig configuration) where T : class where TV : class
{
var producer = new ProducerBuilder <T, TV>(configuration);
producer.SetErrorHandler((_, e) => Devon4NetLogger.Error(new ConsumerException($"Error code {e.Code} : {e.Reason}")));
producer.SetStatisticsHandler((_, json) => Devon4NetLogger.Information($"Statistics: {json}"));
producer.SetLogHandler((c, partitions) => { Devon4NetLogger.Information($"Kafka log handler: [{string.Join(", ", partitions)}]"); });
return(producer);
}
public KafkaProducerContext(ProducerBuilder <TKey, TValue> producerBuilder, IHeadersSerializer headersSerializer, ILogContext logContext,
CancellationToken cancellationToken)
: base(cancellationToken)
{
_logContext = logContext;
_producer = producerBuilder
.SetErrorHandler(OnError)
.Build();
_cancellationTokenSource = new CancellationTokenSource();
HeadersSerializer = headersSerializer;
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaSender"/> class.
/// </summary>
/// <param name="name">The name of the sender.</param>
/// <param name="topic">
/// The topic to produce messages to.
/// </param>
/// <param name="bootstrapServers">
/// List of brokers as a CSV list of broker host or host:port.
/// </param>
/// <param name="messageTimeoutMs">
/// Local message timeout. This value is only enforced locally and limits the time
/// a produced message waits for successful delivery. A time of 0 is infinite. This
/// is the maximum time librdkafka may use to deliver a message (including retries).
/// Delivery error occurs when either the retry count or the message timeout are
/// exceeded.
/// </param>
/// <param name="config">
/// A collection of librdkafka configuration parameters (refer to
/// https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) and parameters
/// specific to this client (refer to: Confluent.Kafka.ConfigPropertyNames).
/// </param>
public KafkaSender(string name, string topic, string bootstrapServers, int messageTimeoutMs = 10000, ProducerConfig config = null)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
Config = config ?? new ProducerConfig();
Config.BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers));
Config.MessageTimeoutMs = Config.MessageTimeoutMs ?? messageTimeoutMs;
var producerBuilder = new ProducerBuilder <Null, string>(Config);
producerBuilder.SetErrorHandler(OnError);
_producer = new Lazy <IProducer <Null, string> >(() => producerBuilder.Build());
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaSender"/> class.
/// </summary>
/// <param name="name">The name of the sender.</param>
/// <param name="topic">The topic to produce messages to.</param>
/// <param name="bootstrapServers">List of brokers as a CSV list of broker host or host:port.</param>
/// <param name="messageTimeoutMs">
/// Local message timeout. This value is only enforced locally and limits the time
/// a produced message waits for successful delivery. A time of 0 is infinite. This
/// is the maximum time librdkafka may use to deliver a message (including retries).
/// Delivery error occurs when either the retry count or the message timeout are
/// exceeded.
/// </param>
public KafkaSender(string name, string topic, string bootstrapServers, int messageTimeoutMs = 10000)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers));
MessageTimeoutMs = messageTimeoutMs;
var config = GetProducerConfig(bootstrapServers, messageTimeoutMs);
var producerBuilder = new ProducerBuilder <string, byte[]>(config);
producerBuilder.SetErrorHandler(OnError);
_producer = new Lazy <IProducer <string, byte[]> >(() => producerBuilder.Build());
}
private static void AddKafkaProducer(this IServiceCollection services)
{
services.AddTransient(p =>
{
var logger = p.GetService <ILogger <IProducer <string, IIntegrationEvent> > >();
var kafkaOptions = p.GetService <IOptions <KafkaOptions> >();
var producerBuilder = new ProducerBuilder <string, IIntegrationEvent>(kafkaOptions.Value.Configuration);
var producer = producerBuilder.SetErrorHandler((_, e) => logger.LogError($"Error: {e?.Reason}", e))
.SetStatisticsHandler((_, json) => logger.LogDebug($"Statistics: {json}"))
.SetValueSerializer(new DefaultValueSerializer <IIntegrationEvent>())
.Build();
return(producer);
});
}
public async Task ProduceAsync(string channelName, MessageBrokerMessage message)
{
var config = new ProducerConfig {
BootstrapServers = _serverAddress
};
var producerBuilder = new ProducerBuilder <string, string>(config);
producerBuilder.SetErrorHandler(
(producer, error) => { _logger.LogWarning(error.Reason); }
);
try
{
while (true)
{
try
{
using (var producer = producerBuilder.Build())
{
// Note: Awaiting the asynchronous produce request below prevents flow of execution
// from proceeding until the acknowledgement from the broker is received (at the
// expense of low throughput).
var deliveryReport = await producer.ProduceAsync(
channelName, KafkaMessage.FromMessageBrokerMessage(message)
);
//producer.Produce(
// channelName, KafkaMessage.FromMessageBrokerMessage(message)
//);
//_logger.LogInformation($"delivered to: {deliveryReport.TopicPartitionOffset}");
return;
}
}
catch (KafkaException e)
{
_logger.LogError($"failed to deliver message: {e.Message} [{e.Error.Code}]");
}
}
}
catch (OperationCanceledException)
{
//
}
}
/// <summary>
/// producer构造器
/// </summary>
/// <returns></returns>
private ProducerBuilder <string, object> CreateProducerBuilder()
{
if (builder == null)
{
lock (this)
{
if (builder == null)
{
ProducerConfig config = new ProducerConfig();
config.BootstrapServers = BootstrapServers;
if (!string.IsNullOrEmpty(SaslUsername))
{
config.SaslUsername = SaslUsername;
config.SaslPassword = SaslPassword;
config.SaslMechanism = SaslMechanism.Plain;
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
}
//List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>();
//config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers));
//if (!string.IsNullOrEmpty(SaslUsername))
//{
// config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT"));
// config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN"));
// config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername));
// config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword));
//}
builder = new ProducerBuilder <string, object>(config);
Action <Delegate, object> tryCatchWrap = (@delegate, arg) =>
{
try
{
@delegate?.DynamicInvoke(arg);
}
catch { }
};
builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason)));
builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e));
builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e)));
builder.SetValueSerializer(new KafkaConverter());
}
}
}
return(builder);
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaSender"/> class.
/// </summary>
/// <param name="name">The name of the sender.</param>
/// <param name="topic">The topic to produce messages to.</param>
/// <param name="producerConfig">The configuration used in creation of the Kafka producer.</param>
public KafkaSender(string name, string topic, ProducerConfig producerConfig)
{
if (producerConfig is null)
{
throw new ArgumentNullException(nameof(producerConfig));
}
Name = name ?? throw new ArgumentNullException(nameof(name));
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
BootstrapServers = producerConfig.BootstrapServers;
MessageTimeoutMs = producerConfig.MessageTimeoutMs;
var producerBuilder = new ProducerBuilder <string, byte[]>(producerConfig);
producerBuilder.SetErrorHandler(OnError);
_producer = new Lazy <IProducer <string, byte[]> >(() => producerBuilder.Build());
}
private void Init(ProducerConfig config)
{
if (_logger.IsDebug)
{
_logger.Debug($"Initializing {Name} type producer for Kafka...");
}
try
{
var producerBuilder = new ProducerBuilder <Null, byte[]>(config);
producerBuilder.SetErrorHandler((s, e) => _logger.Error(e.ToString()));
_producer = producerBuilder.Build();
_initialized = true;
if (_logger.IsDebug)
{
_logger.Debug($"Initialized {Name} type producer for Kafka.");
}
}
catch (Exception e)
{
_logger.Error(e.Message, e);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="KafkaSender"/> class.
/// </summary>
/// <param name="name">The name of the sender.</param>
/// <param name="topic">The topic to produce messages to.</param>
/// <param name="schemaId">
/// The schema ID to have the broker validate messages against. The sender will prepend a leading empty byte
/// and the schema ID to the payload according to the Confluent
/// <a href="https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#wire-format">wire format</a>.
/// </param>
/// <param name="bootstrapServers">List of brokers as a CSV list of broker host or host:port.</param>
/// <param name="messageTimeoutMs">
/// Local message timeout. This value is only enforced locally and limits the time
/// a produced message waits for successful delivery. A time of 0 is infinite. This
/// is the maximum time librdkafka may use to deliver a message (including retries).
/// Delivery error occurs when either the retry count or the message timeout are
/// exceeded.
/// </param>
/// <param name="statisticsIntervalMs">
/// The statistics emit interval in milliseconds. Granularity is 1,000ms. An event handler must be attached to the
/// <see cref="StatisticsEmitted"/> event to receive the statistics data. Setting to 0 disables statistics.
/// </param>
public KafkaSender(string name, string topic, int schemaId, string bootstrapServers, int messageTimeoutMs = Constants.DefaultTimeout,
int statisticsIntervalMs = 0)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Topic = topic ?? throw new ArgumentNullException(nameof(topic));
BootstrapServers = bootstrapServers ?? throw new ArgumentNullException(nameof(bootstrapServers));
MessageTimeoutMs = messageTimeoutMs;
if (schemaId <= 0)
{
throw new ArgumentOutOfRangeException(nameof(schemaId), "Should be greater than 0");
}
SchemaId = schemaId;
var config = GetProducerConfig(bootstrapServers, messageTimeoutMs, statisticsIntervalMs);
var producerBuilder = new ProducerBuilder <string, byte[]>(config);
producerBuilder.SetErrorHandler(OnError);
producerBuilder.SetStatisticsHandler(OnStatisticsEmitted);
_producer = new Lazy <IProducer <string, byte[]> >(() => producerBuilder.Build());
}
public IProducer <byte[], byte[]> GetProducer(ProducerConfig config)
{
ProducerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetProducerBuilder(config);
builder.SetLogHandler(loggerAdapter.LogProduce);
builder.SetErrorHandler(loggerAdapter.ErrorProduce);
if (exposeLibrdKafka)
{
// TODO : test librdkafka statistics with IntegrationTest (WIP see #82)
var producerStatisticsHandler = new ProducerStatisticsHandler(
config.ClientId,
streamConfig.ApplicationId,
(config as StreamizProducerConfig)?.ThreadId,
(config as StreamizProducerConfig)?.Id?.ToString());
producerStatisticsHandler.Register(MetricsRegistry);
builder.SetStatisticsHandler((c, stat) =>
{
var statistics = JsonConvert.DeserializeObject <Statistics>(stat);
producerStatisticsHandler.Publish(statistics);
});
}
return(builder.Build());
}
public string ProduceMessage(int keyId)
{
var message = $"Producer value-{keyId}";
var eventToSend = new Event
{
id = Guid.NewGuid().ToString(),
date = DateTime.Now.ToString("u"),
message = new Message
{
header = new Schemas.Header
{
msg_id = Guid.NewGuid().ToString(),
msg_date = DateTime.Now.ToString("u")
},
Body = message
}
};
try
{
var kafkaConfig = new ProducerConfig {
BootstrapServers = Kafka
};
var prod = new ProducerBuilder <Key, Event>(kafkaConfig);
prod.SetErrorHandler(ProducerErrorHandler);
prod.SetLogHandler(ProducerLogHandler);
using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig))
{
prod.SetKeySerializer(new AvroSerializer <Key>(schemaRegistry));
prod.SetValueSerializer(new AvroSerializer <Event>(schemaRegistry));
using (var p = prod.Build())
{
try
{
var dr = p.ProduceAsync("logs", new Message <Key, Event> {
Key = new Key {
id = keyId
}, Value = eventToSend
}).GetAwaiter().GetResult();
_logger.LogInformation($"Kafka Producer service delivered the message '{message}' to Topic: {dr.Topic}");
}
catch (ProduceException <Key, Event> e)
{
_logger.LogInformation($"Kafka Producer service Delivery failed: {e.Error.Reason}");
}
}
}
}
catch (OperationCanceledException oce)
{
_logger.LogError(oce, "OperationCanceled - Error in delivered message");
}
catch (Exception e)
{
_logger.LogError(e, "Error in delivered message");
throw;
}
return($"The message '{message}' has been sent to the broker.");
}
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken))
{
funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding);
CancellationTokenSource errorCts = new CancellationTokenSource();
CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token);
CancellationToken compositeCancellationToken = compositeCts.Token;
bool aMessageHasBeenProcessed = false;
var cConfig = new ConsumerConfig
{
ClientId = $"{Name}-consumer-{instanceId}",
GroupId = $"{Name}-group",
BootstrapServers = BootstrapServers,
EnableAutoCommit = true,
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Latest
};
if (DebugContext != null)
{
cConfig.Debug = DebugContext;
}
var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig);
if (InKeyDeserializer != null)
{
cBuilder.SetKeyDeserializer(InKeyDeserializer);
}
if (InValueDeserializer != null)
{
cBuilder.SetValueDeserializer(InValueDeserializer);
}
if (Logger != null)
{
cBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
cBuilder.SetErrorHandler((c, e) =>
{
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
// Logger.Log(e);
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var pConfig = new ProducerConfig
{
ClientId = $"{Name}-producer-{instanceId}",
BootstrapServers = BootstrapServers,
EnableIdempotence = true,
LingerMs = 5,
DeliveryReportFields = "none"
};
if (DebugContext != null)
{
pConfig.Debug = DebugContext;
}
var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig);
if (OutKeySerializer != null)
{
pBuilder.SetKeySerializer(OutKeySerializer);
}
if (OutValueSerializer != null)
{
pBuilder.SetValueSerializer(OutValueSerializer);
}
if (Logger != null)
{
pBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
pBuilder.SetErrorHandler((p, e) =>
{
if (e.IsFatal)
{
errorCts.Cancel();
return;
}
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var partitionState = new Dictionary <TopicPartition, PartitionState>();
using (var producer = pBuilder.Build())
using (var consumer = cBuilder.Build())
{
consumer.Subscribe(InputTopic);
try
{
while (true)
{
ConsumeResult <TInKey, TInValue> cr;
try
{
cr = consumer.Consume(compositeCancellationToken);
}
catch (ConsumeException ex)
{
if (ex.Error.Code == ErrorCode.Local_ValueDeserialization)
{
// For an in-depth discussion of what to do in the event of deserialization errors, refer to:
// https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues
if (ConsumeErrorTolerance == ErrorTolerance.All)
{
continue;
}
errorCts.Cancel(); // no error tolerance.
}
Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why?
continue;
}
if (!partitionState.ContainsKey(cr.TopicPartition))
{
partitionState.Add(cr.TopicPartition, new PartitionState(this));
}
partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts);
aMessageHasBeenProcessed = true;
}
}
catch (OperationCanceledException) { }
}
if (errorCts.IsCancellationRequested)
{
throw new Exception("error occured, and we're failing fast.");
}
}
public static void Main(string[] args)
{
var configuration = GetConfiguration(args);
try
{
var prometheusConfig = configuration.GetSection("prometheusMetrics").Get <PrometheusConfig>();
MetricServer metricServer = null;
if (prometheusConfig.Enabled)
{
metricServer = new MetricServer(port: prometheusConfig.Port);
metricServer.Start();
}
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
ProducerBuilder <Null, string> builder = new ProducerBuilder <Null, string>(configuration.GetSection("producerConf").Get <ProducerConfig>());
builder.SetErrorHandler((_, error) =>
{
Console.WriteLine($"An error ocurred producing the event: {error.Reason}");
if (error.IsFatal)
{
Environment.Exit(-1);
}
});
builder.HandleStatistics(new PrometheusProducerStatisticsHandler(new string[] { "application" }, new string[] { "test-producer-statistics" }));
builder.SetKeySerializer(Serializers.Null);
builder.SetValueSerializer(Serializers.Utf8);
using (var producer = builder.Build())
{
Action <DeliveryReport <Null, string> > handler = r =>
{
if (r.Error.IsError)
{
Console.WriteLine($"Delivery Error: {r.Error.Reason}");
}
else
{
Console.WriteLine($"Delivered message to {r.TopicPartitionOffset}");
}
};
int numMessages = 0;
while (!cancellationTokenSource.IsCancellationRequested)
{
try
{
var dr = producer.ProduceAsync(configuration.GetValue <string>("topic"), new Message <Null, string> {
Value = $"message {numMessages}"
});
Console.WriteLine($"Delivered message {numMessages} : {dr.Result.Value}");
Thread.Sleep(1000);
numMessages++;
}
catch (ProduceException <Null, string> e)
{
Console.WriteLine($"Delivery failed: {e.Error.Reason}");
}
}
Console.WriteLine("Exit requested.");
producer.Flush(TimeSpan.FromSeconds(10));
}
Console.WriteLine("Exit requested. Gracefully exiting...");
}
catch (Exception ex)
{
Console.WriteLine("An error occurred while starting up the test.", ex);
Environment.Exit(-2);
}
}
private void DumpMessages <M>(IEnumerable <M> messages, Func <M, Message <K, T> > getMessage, CancellationToken cancellationToken)
{
var producerBuilder = new ProducerBuilder <K, T>(producerConfig);
producerBuilder.SetKeySerializer(keySerializer);
producerBuilder.SetValueSerializer(valueSerializer);
producerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError {
IsFatal = e.IsFatal, Reason = e.Reason
}));
producerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics));
Stopwatch processTime = Stopwatch.StartNew();
using (var p = producerBuilder.Build())
{
foreach (M message in messages)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
Policy.Handle <ProduceException <K, T> >()
.WaitAndRetryForever(retryAttempt => TimeSpan.FromMilliseconds(Math.Min(100 * Math.Pow(2, retryAttempt), 10000)),
(exception, timespan) =>
{
var kafkaException = exception as ProduceException <K, T>;
OnError?.Invoke(new StreamingError
{
IsFatal = kafkaException.Error.IsFatal,
Reason = $"{kafkaException.Error.Reason}. The message with key {kafkaException.DeliveryResult.Key} in topic {kafkaException.DeliveryResult.Topic} will be resent on {timespan.TotalMilliseconds} ms."
});
})
.Execute(() =>
{
if (!cancellationToken.IsCancellationRequested)
{
p.Produce(this.topic, getMessage.Invoke(message),
r =>
{
if (r.Error.IsError)
{
OnError?.Invoke(new StreamingError {
IsFatal = r.Error.IsFatal, Reason = r.Error.Reason
});
}
});
}
});
if (processTime.ElapsedMilliseconds >= CommitTimeoutMs)
{
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
processTime.Restart();
}
}
}
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
}
}
}
