protected virtual void SetValueSerializer(ProducerBuilder <TKey, TValue> producerBuilder)
{
if (ValueSerializer != null)
{
producerBuilder.SetValueSerializer(ValueSerializer);
}
}
public IProducer <string, byte[]> CreateProducer(ClusterSettings clusterSettings, TopicProducerSettings topicProducerSettings)
{
if (topicProducerSettings == null)
{
throw new ArgumentNullException(nameof(topicProducerSettings));
}
if (!topicProducerSettings.Enabled)
{
return(null);
}
var config = clusterSettings.ToClientConfig <ProducerConfig>(c =>
{
c.Acks = (Acks?)topicProducerSettings.Acks;
});
var builder = new ProducerBuilder <string, byte[]>(config)
.SetKeySerializer(Utf8Serializer.Instance);
if (CompressionHandlerFactory.ResolveCompressionHandler(topicProducerSettings.MessageCompressionType, out var compressionHandler))
{
builder.SetValueSerializer(compressionHandler);
}
var producer = builder.Build();
this.producers.Add(producer);
return(producer);
}
private void StartProducer(double seed)
{
Console.WriteLine("Producing {0}", seed);
var producerBuilder = new ProducerBuilder <string, Payload>(_config);
producerBuilder.SetValueSerializer(new ProtoSerializer <Payload>());
using (var producer = producerBuilder.Build())
{
for (long i = 0; i < 1000; i++)
{
string payload = $"{seed}-{i}".ToString(CultureInfo.InvariantCulture);
string key = "sc-" + seed;
var message = new Message <string, Payload>
{
Key = key,
Value = new Payload
{
Key = key,
ProducerId = _settings.ProducerId.ToString(),
Value = payload
}
};
producer.Produce(_settings.Topic, message);
}
producer.Flush(TimeSpan.FromSeconds(30));
}
Console.WriteLine("End of {0}", seed);
}
private void Init(ProducerConfig config)
{
if (_logger.IsDebug)
{
_logger.Debug($"Initializing {Name} type producer for Kafka...");
}
try
{
CachedSchemaRegistryClient schemaRegistry = new CachedSchemaRegistryClient(new[]
{
new KeyValuePair <string, string>(SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl, _schemaRegistryUrl)
});
var blockAvroSerializer = new AvroSerializer <Block>(schemaRegistry).AsSyncOverAsync();
var txAvroSerializer = new AvroSerializer <FullTransaction>(schemaRegistry).AsSyncOverAsync();
ProducerBuilder <Null, Block> blockProducerBuilder = new ProducerBuilder <Null, Block>(config);
blockProducerBuilder.SetValueSerializer(blockAvroSerializer);
blockProducerBuilder.SetErrorHandler((s, e) => _logger.Error(e.ToString()));
ProducerBuilder <Null, FullTransaction> txProducerBuilder = new ProducerBuilder <Null, FullTransaction>(config);
txProducerBuilder.SetValueSerializer(txAvroSerializer);
txProducerBuilder.SetErrorHandler((s, e) => _logger.Error(e.ToString()));
_producerBlocks = blockProducerBuilder.Build();
_producerTransactions = txProducerBuilder.Build();
_initialized = true;
if (_logger.IsDebug)
{
_logger.Debug($"Initialized {Name} type producer for Kafka.");
}
}
catch (Exception e)
{
_logger.Error(e.Message, e);
}
}
/// <summary>
/// 生产
/// </summary>
/// <param name="Key">Message.Key 做消息指定分区投放有用的</param>
/// <param name="Value">Message.Value</param>
/// <param name="Topic">主题</param>
public void Produce(TKey Key, TValue Value, string Topic)
{
var producerBuilder = new ProducerBuilder <TKey, TValue>(ProducerConfig);
producerBuilder.SetValueSerializer(new KafkaConverter <TValue>());//设置序列化方式
using var producer = producerBuilder.Build();
try
{
producer.Produce(Topic, new Message <TKey, TValue>
{
Key = Key,
Value = Value
}, (result) =>
{
if (result.Error.IsError)
{
Logger.Error(LoggerType.KafkaException, $"Topic:{Topic},ServerIp:{KafkaHelper.GetServerIp()},ServerName:{ KafkaHelper.GetServerName()}", null, $"Delivery Error:{result.Error.Reason}");
}
});//Value = JsonConvert.SerializeObject(value)
}
catch (ProduceException <Null, string> ex)
{
Logger.Error(LoggerType.KafkaException, $"Topic:{Topic},Delivery failed: { ex.Error.Reason}", null, ex.Message + ex.StackTrace);
}
}
public ProducerClient(IHostingEnvironment env, IConfiguration globalconf)
{
_globalconf = globalconf;
string sslCaLocation = $@"{ApplicationEnvironment.ApplicationBasePath}{globalconf.GetValue<string>("Kafka:Cert")}";
producerConfig = new ProducerConfig
{
BootstrapServers = globalconf.GetValue <string>("Kafka:BootstrapServers"),
ApiVersionFallbackMs = 0,
//Debug = "security,broker,protocol"
};
if (globalconf.GetValue <bool>("Kafka:UseSSL"))
{
producerConfig.SecurityProtocol = SecurityProtocol.SaslSsl;
producerConfig.SaslMechanism = SaslMechanism.Plain;
producerConfig.SslCaLocation = sslCaLocation;
}
if (globalconf.GetValue <bool>("Kafka:RequireLogin"))
{
producerConfig.SaslUsername = globalconf.GetValue <string>("Kafka:Username");
producerConfig.SaslPassword = globalconf.GetValue <string>("Kafka:Password");
}
//if (env.IsDevelopment())
//{
// producerConfig = new ProducerConfig
// {
// BootstrapServers = globalconf.GetValue<string>("Kafka:BootstrapServers"),
// SecurityProtocol = SecurityProtocol.SaslSsl,
// SaslMechanism = SaslMechanism.Plain,
// SaslUsername = globalconf.GetValue<string>("Kafka:Username"),
// SaslPassword = globalconf.GetValue<string>("Kafka:Password"),
// SslCaLocation = sslCaLocation,
// ApiVersionFallbackMs = 0,
// Debug = "security,broker,protocol"
// };
// producerConfig = new ProducerConfig { BootstrapServers = "localhost:9092" };
//}
//else
//{
// producerConfig = new ProducerConfig
// {
// BootstrapServers = globalconf.GetValue<string>("Kafka:BootstrapServers"),
// SecurityProtocol = SecurityProtocol.SaslSsl,
// SaslMechanism = SaslMechanism.Plain,
// SaslUsername = globalconf.GetValue<string>("Kafka:Username"),
// SaslPassword = globalconf.GetValue<string>("Kafka:Password"),
// SslCaLocation = sslCaLocation,
// ApiVersionFallbackMs = 0,
// };
//}
var producerBuilder = new ProducerBuilder <Null, T>(producerConfig);
producerBuilder.SetValueSerializer(new KafkaByteSerializer <T>());
producer = producerBuilder.Build();
}
/// <summary>
/// Sets the Message Value serializers.
/// </summary>
/// <param name="builder">The <see cref="ProducerBuilder{TKey, TValue}"/>.</param>
/// <param name="serializers">The serializers to use.</param>
/// <returns>The <see cref="ProducerBuilder{TKey, TValue}"/>.</returns>
public static ProducerBuilder <TKey, TValue> SetValueSerializer <TKey, TValue>(this ProducerBuilder <TKey, TValue> builder, Serializers <TKey, TValue> serializers)
{
if (builder == null)
{
throw new ArgumentNullException(nameof(builder));
}
if (serializers.ValueSerializer != null)
{
builder.SetValueSerializer(serializers.ValueSerializer);
}
else if (serializers.AsyncValueSerializer != null)
{
builder.SetValueSerializer(serializers.AsyncValueSerializer);
}
return(builder);
}
/// <summary>
/// Set the message value serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// A client to use for Schema Registry operations. The client should only be disposed
/// after the producer; the serializer will use it to request schemas as messages are being
/// produced.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
public static ProducerBuilder <TKey, TValue> SetAvroValueSerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
ISchemaRegistryClient registryClient,
bool registerAutomatically = false,
Func <SerializationContext, string> subjectNameBuilder = null
) => producerBuilder.SetValueSerializer(new AsyncSchemaRegistrySerializer <TValue>(
registryClient,
registerAutomatically: registerAutomatically,
subjectNameBuilder: subjectNameBuilder
));
/// <summary>
/// Set the message value serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryConfiguration">
/// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is
/// highly recommended.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
public static ProducerBuilder <TKey, TValue> SetAvroValueSerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
IEnumerable <KeyValuePair <string, string> > registryConfiguration,
bool registerAutomatically = false,
Func <SerializationContext, string> subjectNameBuilder = null
) => producerBuilder.SetValueSerializer(new AsyncSchemaRegistrySerializer <TValue>(
registryConfiguration,
registerAutomatically: registerAutomatically,
subjectNameBuilder: subjectNameBuilder
));
protected virtual IProducer <string, T> CreateProducer()
{
ProducerBuilder <string, T> producerBuilder = new ProducerBuilder <string, T>(ProducerConfig);
if (Serializer != null)
{
producerBuilder.SetValueSerializer(Serializer);
}
return(producerBuilder.Build());
}
public ProducerConnectionBuilder <TKey, TValue> WithSchemaRegistry(string url)
{
if (typeof(TKey) == typeof(string) || typeof(TValue) == typeof(string))
{
return(this);
}
var schemaRegistryConfig = new SchemaRegistryConfig()
{
SchemaRegistryUrl = url
};
var schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
var serializerConfig = new AvroSerializerConfig();
producerBuilder.SetKeySerializer(new AvroSerializer <TKey>(schemaRegistryClient, serializerConfig));
producerBuilder.SetValueSerializer(new AvroSerializer <TValue>(schemaRegistryClient, serializerConfig));
return(this);
}
public KafkaProducer(
ProducerConfig config,
string avroSchema,
ILogger logger)
{
this.logger = logger;
var builder = new ProducerBuilder <TKey, TValue>(config);
IAsyncSerializer <TValue> asyncValueSerializer = null;
ISerializer <TValue> valueSerializer = null;
IAsyncSerializer <TKey> keySerializer = null;
if (!string.IsNullOrEmpty(avroSchema))
{
var schemaRegistry = new LocalSchemaRegistry(avroSchema);
asyncValueSerializer = new AvroSerializer <TValue>(schemaRegistry);
}
else
{
if (typeof(Google.Protobuf.IMessage).IsAssignableFrom(typeof(TValue)))
{
// protobuf: need to create using reflection due to generic requirements in ProtobufSerializer
valueSerializer = (ISerializer <TValue>)Activator.CreateInstance(typeof(ProtobufSerializer <>).MakeGenericType(typeof(TValue)));
}
}
if (asyncValueSerializer != null)
{
builder.SetValueSerializer(asyncValueSerializer);
}
else if (valueSerializer != null)
{
builder.SetValueSerializer(valueSerializer);
}
if (keySerializer != null)
{
builder.SetKeySerializer(keySerializer);
}
this.producer = builder.Build();
}
private IProducer <byte[], T> InitializeProducer()
{
var config = new ProducerConfig {
BootstrapServers = string.Join(',', _kafkaConfig.CurrentValue.Brokers)
};
var producerBuilder = new ProducerBuilder <byte[], T>(config);
if (typeof(T) == typeof(MessageData.MessageData))
{
producerBuilder.SetValueSerializer((ISerializer <T>) new MessageDataSerializer());
}
return(producerBuilder.Build());
}
void Start()
{
var config = new ProducerConfig
{
BootstrapServers = "localhost:9092",
ClientId = Dns.GetHostName(),
Acks = 0
};
var pb = new ProducerBuilder <int, Vector3>(config);
pb.SetValueSerializer(new Vector3Serializer());
_producer = pb.Build();
}
/// <summary>
/// producer构造器
/// </summary>
/// <returns></returns>
private ProducerBuilder <string, object> CreateProducerBuilder()
{
if (builder == null)
{
lock (this)
{
if (builder == null)
{
ProducerConfig config = new ProducerConfig();
config.BootstrapServers = BootstrapServers;
if (!string.IsNullOrEmpty(SaslUsername))
{
config.SaslUsername = SaslUsername;
config.SaslPassword = SaslPassword;
config.SaslMechanism = SaslMechanism.Plain;
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
}
//List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>();
//config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers));
//if (!string.IsNullOrEmpty(SaslUsername))
//{
// config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT"));
// config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN"));
// config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername));
// config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword));
//}
builder = new ProducerBuilder <string, object>(config);
Action <Delegate, object> tryCatchWrap = (@delegate, arg) =>
{
try
{
@delegate?.DynamicInvoke(arg);
}
catch { }
};
builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason)));
builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e));
builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e)));
builder.SetValueSerializer(new KafkaConverter());
}
}
}
return(builder);
}
/// <summary>
/// 初始化
/// </summary>
/// <param name="_config_">服务器、Topic、用户名和密码</param>
public KafkaProducer(KafkaProducerConfigForCredit _config_, ISerializer <T> serializer = null)
{
this.TopicName = _config_.TopicName;
producerConfig = new ProducerConfig
{
BootstrapServers = _config_.BrokerServers,
SaslUsername = _config_.SaslUsername,
SaslPassword = _config_.SaslPassword,
SaslMechanism = _config_.SaslMechanism,
SecurityProtocol = _config_.SecurityProtocol
};
var producerBuilder = new ProducerBuilder <string, T>(producerConfig);
if (serializer != null)
{
producerBuilder = producerBuilder.SetValueSerializer(serializer);
}
_producer = producerBuilder.Build();
}
/// <summary>
/// 生产异步
/// </summary>
/// <param name="Key">Message.Key</param>
/// <param name="Value">Message.Value</param>
/// <param name="Topic">主题</param>
/// <returns></returns>
public async Task ProduceAsync(TKey Key, TValue Value, string Topic)
{
var producerBuilder = new ProducerBuilder <TKey, TValue>(ProducerConfig);
producerBuilder.SetValueSerializer(new KafkaConverter <TValue>());
using var producer = producerBuilder.Build();
try
{
var dr = await producer.ProduceAsync(Topic, new Message <TKey, TValue>
{
Key = Key,
Value = Value
});
//Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'");
}
catch (ProduceException <Null, string> ex)
{
Logger.Error(LoggerType.KafkaException, $"Topic:{Topic},ServerIp:{KafkaHelper.GetServerIp()},ServerName:{ KafkaHelper.GetServerName()},Delivery failed: { ex.Error.Reason}", null, ex.Message + ex.StackTrace);
}
}
/// <summary>
/// 普通初始化
/// </summary>
/// <param name="_config">如果有用户名和密码,请放在KafkaConsumerConfig的ClientConfig属性中</param>
public KafkaProducer(KafkaProducerConfig _config_, ISerializer <T> serializer = null)
{
this.TopicName = _config_.TopicName;
if (_config_.ClientConfig?.Count > 0)
{
producerConfig = new ProducerConfig(_config_.ClientConfig)
{
BootstrapServers = _config_.BrokerServers
};
}
else
{
producerConfig = new ProducerConfig {
BootstrapServers = _config_.BrokerServers
};
}
var producerBuilder = new ProducerBuilder <string, T>(producerConfig);
if (serializer != null)
{
_producer = producerBuilder.SetValueSerializer(serializer).Build();
}
_producer = producerBuilder.Build();
}
public string ProduceMessage(int keyId)
{
var message = $"Producer value-{keyId}";
var eventToSend = new Event
{
id = Guid.NewGuid().ToString(),
date = DateTime.Now.ToString("u"),
message = new Message
{
header = new Schemas.Header
{
msg_id = Guid.NewGuid().ToString(),
msg_date = DateTime.Now.ToString("u")
},
Body = message
}
};
try
{
var kafkaConfig = new ProducerConfig {
BootstrapServers = Kafka
};
var prod = new ProducerBuilder <Key, Event>(kafkaConfig);
prod.SetErrorHandler(ProducerErrorHandler);
prod.SetLogHandler(ProducerLogHandler);
using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig))
{
prod.SetKeySerializer(new AvroSerializer <Key>(schemaRegistry));
prod.SetValueSerializer(new AvroSerializer <Event>(schemaRegistry));
using (var p = prod.Build())
{
try
{
var dr = p.ProduceAsync("logs", new Message <Key, Event> {
Key = new Key {
id = keyId
}, Value = eventToSend
}).GetAwaiter().GetResult();
_logger.LogInformation($"Kafka Producer service delivered the message '{message}' to Topic: {dr.Topic}");
}
catch (ProduceException <Key, Event> e)
{
_logger.LogInformation($"Kafka Producer service Delivery failed: {e.Error.Reason}");
}
}
}
}
catch (OperationCanceledException oce)
{
_logger.LogError(oce, "OperationCanceled - Error in delivered message");
}
catch (Exception e)
{
_logger.LogError(e, "Error in delivered message");
throw;
}
return($"The message '{message}' has been sent to the broker.");
}
public static void Main(string[] args)
{
var configuration = GetConfiguration(args);
try
{
var prometheusConfig = configuration.GetSection("prometheusMetrics").Get <PrometheusConfig>();
MetricServer metricServer = null;
if (prometheusConfig.Enabled)
{
metricServer = new MetricServer(port: prometheusConfig.Port);
metricServer.Start();
}
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
ProducerBuilder <Null, string> builder = new ProducerBuilder <Null, string>(configuration.GetSection("producerConf").Get <ProducerConfig>());
builder.SetErrorHandler((_, error) =>
{
Console.WriteLine($"An error ocurred producing the event: {error.Reason}");
if (error.IsFatal)
{
Environment.Exit(-1);
}
});
builder.HandleStatistics(new PrometheusProducerStatisticsHandler(new string[] { "application" }, new string[] { "test-producer-statistics" }));
builder.SetKeySerializer(Serializers.Null);
builder.SetValueSerializer(Serializers.Utf8);
using (var producer = builder.Build())
{
Action <DeliveryReport <Null, string> > handler = r =>
{
if (r.Error.IsError)
{
Console.WriteLine($"Delivery Error: {r.Error.Reason}");
}
else
{
Console.WriteLine($"Delivered message to {r.TopicPartitionOffset}");
}
};
int numMessages = 0;
while (!cancellationTokenSource.IsCancellationRequested)
{
try
{
var dr = producer.ProduceAsync(configuration.GetValue <string>("topic"), new Message <Null, string> {
Value = $"message {numMessages}"
});
Console.WriteLine($"Delivered message {numMessages} : {dr.Result.Value}");
Thread.Sleep(1000);
numMessages++;
}
catch (ProduceException <Null, string> e)
{
Console.WriteLine($"Delivery failed: {e.Error.Reason}");
}
}
Console.WriteLine("Exit requested.");
producer.Flush(TimeSpan.FromSeconds(10));
}
Console.WriteLine("Exit requested. Gracefully exiting...");
}
catch (Exception ex)
{
Console.WriteLine("An error occurred while starting up the test.", ex);
Environment.Exit(-2);
}
}
private void DumpMessages <M>(IEnumerable <M> messages, Func <M, Message <K, T> > getMessage, CancellationToken cancellationToken)
{
var producerBuilder = new ProducerBuilder <K, T>(producerConfig);
producerBuilder.SetKeySerializer(keySerializer);
producerBuilder.SetValueSerializer(valueSerializer);
producerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError {
IsFatal = e.IsFatal, Reason = e.Reason
}));
producerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics));
Stopwatch processTime = Stopwatch.StartNew();
using (var p = producerBuilder.Build())
{
foreach (M message in messages)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
Policy.Handle <ProduceException <K, T> >()
.WaitAndRetryForever(retryAttempt => TimeSpan.FromMilliseconds(Math.Min(100 * Math.Pow(2, retryAttempt), 10000)),
(exception, timespan) =>
{
var kafkaException = exception as ProduceException <K, T>;
OnError?.Invoke(new StreamingError
{
IsFatal = kafkaException.Error.IsFatal,
Reason = $"{kafkaException.Error.Reason}. The message with key {kafkaException.DeliveryResult.Key} in topic {kafkaException.DeliveryResult.Topic} will be resent on {timespan.TotalMilliseconds} ms."
});
})
.Execute(() =>
{
if (!cancellationToken.IsCancellationRequested)
{
p.Produce(this.topic, getMessage.Invoke(message),
r =>
{
if (r.Error.IsError)
{
OnError?.Invoke(new StreamingError {
IsFatal = r.Error.IsFatal, Reason = r.Error.Reason
});
}
});
}
});
if (processTime.ElapsedMilliseconds >= CommitTimeoutMs)
{
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
processTime.Restart();
}
}
}
p.Flush(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
OnCommit?.Invoke();
}
}
}
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken))
{
funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding);
CancellationTokenSource errorCts = new CancellationTokenSource();
CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token);
CancellationToken compositeCancellationToken = compositeCts.Token;
bool aMessageHasBeenProcessed = false;
var cConfig = new ConsumerConfig
{
ClientId = $"{Name}-consumer-{instanceId}",
GroupId = $"{Name}-group",
BootstrapServers = BootstrapServers,
EnableAutoCommit = true,
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Latest
};
if (DebugContext != null)
{
cConfig.Debug = DebugContext;
}
var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig);
if (InKeyDeserializer != null)
{
cBuilder.SetKeyDeserializer(InKeyDeserializer);
}
if (InValueDeserializer != null)
{
cBuilder.SetValueDeserializer(InValueDeserializer);
}
if (Logger != null)
{
cBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
cBuilder.SetErrorHandler((c, e) =>
{
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
// Logger.Log(e);
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var pConfig = new ProducerConfig
{
ClientId = $"{Name}-producer-{instanceId}",
BootstrapServers = BootstrapServers,
EnableIdempotence = true,
LingerMs = 5,
DeliveryReportFields = "none"
};
if (DebugContext != null)
{
pConfig.Debug = DebugContext;
}
var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig);
if (OutKeySerializer != null)
{
pBuilder.SetKeySerializer(OutKeySerializer);
}
if (OutValueSerializer != null)
{
pBuilder.SetValueSerializer(OutValueSerializer);
}
if (Logger != null)
{
pBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
pBuilder.SetErrorHandler((p, e) =>
{
if (e.IsFatal)
{
errorCts.Cancel();
return;
}
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var partitionState = new Dictionary <TopicPartition, PartitionState>();
using (var producer = pBuilder.Build())
using (var consumer = cBuilder.Build())
{
consumer.Subscribe(InputTopic);
try
{
while (true)
{
ConsumeResult <TInKey, TInValue> cr;
try
{
cr = consumer.Consume(compositeCancellationToken);
}
catch (ConsumeException ex)
{
if (ex.Error.Code == ErrorCode.Local_ValueDeserialization)
{
// For an in-depth discussion of what to do in the event of deserialization errors, refer to:
// https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues
if (ConsumeErrorTolerance == ErrorTolerance.All)
{
continue;
}
errorCts.Cancel(); // no error tolerance.
}
Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why?
continue;
}
if (!partitionState.ContainsKey(cr.TopicPartition))
{
partitionState.Add(cr.TopicPartition, new PartitionState(this));
}
partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts);
aMessageHasBeenProcessed = true;
}
}
catch (OperationCanceledException) { }
}
if (errorCts.IsCancellationRequested)
{
throw new Exception("error occured, and we're failing fast.");
}
}
/// <summary>
/// Set the message value serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="serializerBuilder">
/// A serializer builder.
/// </param>
/// <param name="subject">
/// The subject of the schema that should be used to serialize values.
/// </param>
/// <param name="version">
/// The version of the subject to be resolved.
/// </param>
public static async Task <ProducerBuilder <TKey, TValue> > SetAvroValueSerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
SchemaRegistrySerializerBuilder serializerBuilder,
string subject,
int version
) => producerBuilder.SetValueSerializer(await serializerBuilder.Build <TValue>(subject, version));
/// <summary>
/// Set the message value serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="serializerBuilder">
/// A serializer builder.
/// </param>
/// <param name="subject">
/// The subject of the schema that should be used to serialize values. The latest version
/// of the subject will be resolved.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register a schema that matches <typeparamref name="TValue" />
/// if one does not already exist.
/// </param>
public static async Task <ProducerBuilder <TKey, TValue> > SetAvroValueSerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
SchemaRegistrySerializerBuilder serializerBuilder,
string subject,
bool registerAutomatically = false
) => producerBuilder.SetValueSerializer(await serializerBuilder.Build <TValue>(subject, registerAutomatically));
/// <summary>
/// Set the message value serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="serializerBuilder">
/// A serializer builder.
/// </param>
/// <param name="id">
/// The ID of the schema that should be used to serialize values.
/// </param>
public static async Task <ProducerBuilder <TKey, TValue> > SetAvroValueSerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
SchemaRegistrySerializerBuilder serializerBuilder,
int id
) => producerBuilder.SetValueSerializer(await serializerBuilder.Build <TValue>(id));
/// <summary>
/// 高级初始化的方法
/// </summary>
/// <param name="clientConfig">通常只需要设置：clientConfig的BootstrapServers、SaslUsername、SaslPassword、SaslMechanism、SecurityProtocol等属性即可</param>
/// <param name="TopicName"></param>
public KafkaProducer(KafkaClientConfig clientConfig, string TopicName, ISerializer <T> serializer = null)
{
this.TopicName = TopicName;
producerConfig = new ProducerConfig
{
#region 常规选项（服务器地址、用户名和密码）
BootstrapServers = clientConfig.BootstrapServers,
SaslUsername = clientConfig.SaslUsername,
SaslPassword = clientConfig.SaslPassword,
SaslMechanism = clientConfig.SaslMechanism,
SecurityProtocol = clientConfig.SecurityProtocol,
#endregion
#region 高级选项设置（你可以不用管）
Acks = clientConfig.Acks,
ApiVersionFallbackMs = clientConfig.ApiVersionFallbackMs,
ApiVersionRequest = clientConfig.ApiVersionRequest,
ApiVersionRequestTimeoutMs = clientConfig.ApiVersionRequestTimeoutMs,
BrokerAddressFamily = clientConfig.BrokerAddressFamily,
BrokerAddressTtl = clientConfig.BrokerAddressTtl,
BrokerVersionFallback = clientConfig.BrokerVersionFallback,
ClientId = clientConfig.ClientId,
Debug = clientConfig.Debug,
MaxInFlight = clientConfig.MaxInFlight,
LogConnectionClose = clientConfig.LogConnectionClose,
MessageCopyMaxBytes = clientConfig.MessageCopyMaxBytes,
EnableSaslOauthbearerUnsecureJwt = clientConfig.EnableSaslOauthbearerUnsecureJwt,
LogQueue = clientConfig.LogQueue,
InternalTerminationSignal = clientConfig.InternalTerminationSignal,
EnableSslCertificateVerification = clientConfig.EnableSslCertificateVerification,
LogThreadName = clientConfig.LogThreadName,
MessageMaxBytes = clientConfig.MessageMaxBytes,
MetadataMaxAgeMs = clientConfig.MetadataMaxAgeMs,
MetadataRequestTimeoutMs = clientConfig.MetadataRequestTimeoutMs,
PluginLibraryPaths = clientConfig.PluginLibraryPaths,
ReceiveMessageMaxBytes = clientConfig.ReceiveMessageMaxBytes,
ReconnectBackoffMaxMs = clientConfig.ReconnectBackoffMaxMs,
ReconnectBackoffMs = clientConfig.ReconnectBackoffMs,
SaslKerberosKeytab = clientConfig.SaslKerberosKeytab,
SaslKerberosKinitCmd = clientConfig.SaslKerberosKinitCmd,
SaslKerberosMinTimeBeforeRelogin = clientConfig.SaslKerberosMinTimeBeforeRelogin,
SaslKerberosPrincipal = clientConfig.SaslKerberosPrincipal,
SaslKerberosServiceName = clientConfig.SaslKerberosServiceName,
SaslOauthbearerConfig = clientConfig.SaslOauthbearerConfig,
SocketKeepaliveEnable = clientConfig.SocketKeepaliveEnable,
SocketMaxFails = clientConfig.SocketMaxFails,
SocketNagleDisable = clientConfig.SocketNagleDisable,
SocketReceiveBufferBytes = clientConfig.SocketReceiveBufferBytes,
SocketSendBufferBytes = clientConfig.SocketSendBufferBytes,
SocketTimeoutMs = clientConfig.SocketTimeoutMs,
SslCaLocation = clientConfig.SslCaLocation,
SslCertificateLocation = clientConfig.SslCertificateLocation,
SslCertificatePem = clientConfig.SslCertificatePem,
SslCipherSuites = clientConfig.SslCipherSuites,
SslCrlLocation = clientConfig.SslCrlLocation,
SslCurvesList = clientConfig.SslCurvesList,
SslEndpointIdentificationAlgorithm = clientConfig.SslEndpointIdentificationAlgorithm,
SslKeyLocation = clientConfig.SslKeyLocation,
SslKeyPassword = clientConfig.SslKeyPassword,
SslKeyPem = clientConfig.SslKeyPem,
SslKeystoreLocation = clientConfig.SslKeystoreLocation,
SslKeystorePassword = clientConfig.SslKeystorePassword,
SslSigalgsList = clientConfig.SslSigalgsList,
StatisticsIntervalMs = clientConfig.StatisticsIntervalMs,
TopicMetadataRefreshSparse = clientConfig.TopicMetadataRefreshSparse,
TopicBlacklist = clientConfig.TopicBlacklist,
TopicMetadataRefreshFastIntervalMs = clientConfig.TopicMetadataRefreshFastIntervalMs,
TopicMetadataRefreshIntervalMs = clientConfig.TopicMetadataRefreshIntervalMs
#endregion
};
var producerBuilder = new ProducerBuilder <string, T>(producerConfig);
if (serializer != null)
{
producerBuilder = producerBuilder.SetValueSerializer(serializer);
}
_producer = producerBuilder.Build();
}
