public IProducer <Null, byte[]> CreateProducer(
KafkaQueueConfiguration config,
Action <IProducer <Null, byte[]>, LogMessage> logHandler = null,
Action <IProducer <Null, byte[]>, Error> errorHandler = null)
{
config.ThrowIfNull(nameof(config));
var builder = new ProducerBuilder <Null, byte[]>(new ProducerConfig
{
BootstrapServers = $"{config.Server}",
SaslKerberosKeytab = config.KeyTab,
SaslKerberosPrincipal = config.User,
SaslKerberosServiceName = config.ServiceName,
SecurityProtocol = config.Protocol,
SaslMechanism = config.Mechanism,
Debug = config.Debug
});
if (logHandler != null)
{
builder.SetLogHandler(logHandler);
}
if (errorHandler != null)
{
builder.SetErrorHandler(errorHandler);
}
return(builder.Build());
}
public IProducer <byte[], byte[]> GetProducer(ProducerConfig config)
{
ProducerBuilder <byte[], byte[]> builder = new ProducerBuilder <byte[], byte[]>(config);
builder.SetLogHandler(loggerAdapter.LogProduce);
builder.SetErrorHandler(loggerAdapter.ErrorProduce);
return(builder.Build());
}
private static ProducerBuilder <T, TV> GetProducerBuilderInstance <T, TV>(ProducerConfig configuration) where T : class where TV : class
{
var producer = new ProducerBuilder <T, TV>(configuration);
producer.SetErrorHandler((_, e) => Devon4NetLogger.Error(new ConsumerException($"Error code {e.Code} : {e.Reason}")));
producer.SetStatisticsHandler((_, json) => Devon4NetLogger.Information($"Statistics: {json}"));
producer.SetLogHandler((c, partitions) => { Devon4NetLogger.Information($"Kafka log handler: [{string.Join(", ", partitions)}]"); });
return(producer);
}
private async void Produce_Click(object sender, EventArgs e)
{
if (producer == null)
{
var config = new ProducerConfig {
BootstrapServers = bootstrapServers.Text
};
if (!string.IsNullOrWhiteSpace(username.Text) &&
!string.IsNullOrWhiteSpace(password.Text))
{
config.SaslUsername = username.Text;
config.SaslPassword = password.Text;
if (securityProtocol.SelectedItem != null)
{
config.SecurityProtocol = ((ComboboxItem <SecurityProtocol>)securityProtocol.SelectedItem).Value;
if (config.SecurityProtocol == SecurityProtocol.SaslSsl ||
config.SecurityProtocol == SecurityProtocol.Ssl)
{
config.SslCaLocation = caCertificateFileLocation.Text;
}
}
if (saslMechanism.SelectedItem != null)
{
config.SaslMechanism = ((ComboboxItem <SaslMechanism>)saslMechanism.SelectedItem).Value;
}
}
config.Debug = "all";
try
{
var builder = new ProducerBuilder <Null, string>(config);
builder.SetLogHandler((producer, message) =>
{
// add logging
});
producer = new ProducerBuilder <Null, string>(config).Build();
}
catch (ProduceException <Null, string> ex)
{
MessageBox.Show(this, ex.Message);
return;
}
}
try
{
var dr = await producer.ProduceAsync(produceToTopic.Text,
new Message <Null, string> {
Value = input.TextValue
}).ConfigureAwait(true);
status.Text = $"Delivered to '{dr.TopicPartitionOffset}'";
}
catch (ProduceException <Null, string> ex)
{
MessageBox.Show(this, $"Delivery failed: {ex.Error.Reason}");
}
}
private IProducer <byte[], byte[]> CreateBaseProducer(ProducerConfig producerConfig)
{
var builder = new ProducerBuilder <byte[], byte[]>(producerConfig);
ILogger logger = this.loggerProvider.CreateLogger("Kafka");
builder.SetLogHandler((_, m) =>
{
logger.Log((LogLevel)m.LevelAs(LogLevelType.MicrosoftExtensionsLogging), $"Libkafka: {m?.Message}");
});
return(builder.Build());
}
/// <summary>
/// producer构造器
/// </summary>
/// <returns></returns>
private ProducerBuilder <string, object> CreateProducerBuilder()
{
if (builder == null)
{
lock (this)
{
if (builder == null)
{
ProducerConfig config = new ProducerConfig();
config.BootstrapServers = BootstrapServers;
if (!string.IsNullOrEmpty(SaslUsername))
{
config.SaslUsername = SaslUsername;
config.SaslPassword = SaslPassword;
config.SaslMechanism = SaslMechanism.Plain;
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
}
//List<KeyValuePair<string, string>> config = new List<KeyValuePair<string, string>>();
//config.Add(new KeyValuePair<string, string>("bootstrap.servers", BootstrapServers));
//if (!string.IsNullOrEmpty(SaslUsername))
//{
// config.Add(new KeyValuePair<string, string>("security.protocol", "SASL_PLAINTEXT"));
// config.Add(new KeyValuePair<string, string>("sasl.mechanism", "PLAIN"));
// config.Add(new KeyValuePair<string, string>("sasl.username", SaslUsername));
// config.Add(new KeyValuePair<string, string>("sasl.password", SaslPassword));
//}
builder = new ProducerBuilder <string, object>(config);
Action <Delegate, object> tryCatchWrap = (@delegate, arg) =>
{
try
{
@delegate?.DynamicInvoke(arg);
}
catch { }
};
builder.SetErrorHandler((p, e) => tryCatchWrap(ErrorHandler, new Exception(e.Reason)));
builder.SetStatisticsHandler((p, e) => tryCatchWrap(StatisticsHandler, e));
builder.SetLogHandler((p, e) => tryCatchWrap(LogHandler, new KafkaLogMessage(e)));
builder.SetValueSerializer(new KafkaConverter());
}
}
}
return(builder);
}
/// <inheritdoc cref="IConfluentProducerBuilder.Build" />
public IProducer <byte[]?, byte[]?> Build()
{
if (_config == null)
{
throw new InvalidOperationException("SetConfig must be called to provide the producer configuration.");
}
var builder = new ProducerBuilder <byte[]?, byte[]?>(_config);
if (_statisticsHandler != null)
{
builder.SetStatisticsHandler(_statisticsHandler);
}
if (_logHandler != null)
{
builder.SetLogHandler(_logHandler);
}
return(builder.Build());
}
public IProducer <byte[], byte[]> GetProducer(ProducerConfig config)
{
ProducerBuilder <byte[], byte[]> builder = builderKafkaHandler.GetProducerBuilder(config);
builder.SetLogHandler(loggerAdapter.LogProduce);
builder.SetErrorHandler(loggerAdapter.ErrorProduce);
if (exposeLibrdKafka)
{
// TODO : test librdkafka statistics with IntegrationTest (WIP see #82)
var producerStatisticsHandler = new ProducerStatisticsHandler(
config.ClientId,
streamConfig.ApplicationId,
(config as StreamizProducerConfig)?.ThreadId,
(config as StreamizProducerConfig)?.Id?.ToString());
producerStatisticsHandler.Register(MetricsRegistry);
builder.SetStatisticsHandler((c, stat) =>
{
var statistics = JsonConvert.DeserializeObject <Statistics>(stat);
producerStatisticsHandler.Publish(statistics);
});
}
return(builder.Build());
}
public string ProduceMessage(int keyId)
{
var message = $"Producer value-{keyId}";
var eventToSend = new Event
{
id = Guid.NewGuid().ToString(),
date = DateTime.Now.ToString("u"),
message = new Message
{
header = new Schemas.Header
{
msg_id = Guid.NewGuid().ToString(),
msg_date = DateTime.Now.ToString("u")
},
Body = message
}
};
try
{
var kafkaConfig = new ProducerConfig {
BootstrapServers = Kafka
};
var prod = new ProducerBuilder <Key, Event>(kafkaConfig);
prod.SetErrorHandler(ProducerErrorHandler);
prod.SetLogHandler(ProducerLogHandler);
using (var schemaRegistry = new CachedSchemaRegistryClient(_schemaRegistryConfig))
{
prod.SetKeySerializer(new AvroSerializer <Key>(schemaRegistry));
prod.SetValueSerializer(new AvroSerializer <Event>(schemaRegistry));
using (var p = prod.Build())
{
try
{
var dr = p.ProduceAsync("logs", new Message <Key, Event> {
Key = new Key {
id = keyId
}, Value = eventToSend
}).GetAwaiter().GetResult();
_logger.LogInformation($"Kafka Producer service delivered the message '{message}' to Topic: {dr.Topic}");
}
catch (ProduceException <Key, Event> e)
{
_logger.LogInformation($"Kafka Producer service Delivery failed: {e.Error.Reason}");
}
}
}
}
catch (OperationCanceledException oce)
{
_logger.LogError(oce, "OperationCanceled - Error in delivered message");
}
catch (Exception e)
{
_logger.LogError(e, "Error in delivered message");
throw;
}
return($"The message '{message}' has been sent to the broker.");
}
public void Start(string instanceId, CancellationToken cancellationToken = default(CancellationToken))
{
funcExecSemaphore = new Semaphore(MaxOutstanding, MaxOutstanding);
CancellationTokenSource errorCts = new CancellationTokenSource();
CancellationTokenSource compositeCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCts.Token);
CancellationToken compositeCancellationToken = compositeCts.Token;
bool aMessageHasBeenProcessed = false;
var cConfig = new ConsumerConfig
{
ClientId = $"{Name}-consumer-{instanceId}",
GroupId = $"{Name}-group",
BootstrapServers = BootstrapServers,
EnableAutoCommit = true,
EnableAutoOffsetStore = false,
AutoOffsetReset = AutoOffsetReset.Latest
};
if (DebugContext != null)
{
cConfig.Debug = DebugContext;
}
var cBuilder = new ConsumerBuilder <TInKey, TInValue>(cConfig);
if (InKeyDeserializer != null)
{
cBuilder.SetKeyDeserializer(InKeyDeserializer);
}
if (InValueDeserializer != null)
{
cBuilder.SetValueDeserializer(InValueDeserializer);
}
if (Logger != null)
{
cBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
cBuilder.SetErrorHandler((c, e) =>
{
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
// Logger.Log(e);
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(c.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var pConfig = new ProducerConfig
{
ClientId = $"{Name}-producer-{instanceId}",
BootstrapServers = BootstrapServers,
EnableIdempotence = true,
LingerMs = 5,
DeliveryReportFields = "none"
};
if (DebugContext != null)
{
pConfig.Debug = DebugContext;
}
var pBuilder = new ProducerBuilder <TOutKey, TOutValue>(pConfig);
if (OutKeySerializer != null)
{
pBuilder.SetKeySerializer(OutKeySerializer);
}
if (OutValueSerializer != null)
{
pBuilder.SetValueSerializer(OutValueSerializer);
}
if (Logger != null)
{
pBuilder.SetLogHandler((_, m) =>
{
Logger(m);
});
}
pBuilder.SetErrorHandler((p, e) =>
{
if (e.IsFatal)
{
errorCts.Cancel();
return;
}
if (e.Code == ErrorCode.Local_AllBrokersDown ||
e.Code == ErrorCode.Local_Authentication)
{
if (!aMessageHasBeenProcessed)
{
errorCts.Cancel();
return;
}
}
if (Logger != null)
{
Logger(new LogMessage(p.Name, SyslogLevel.Error, "unknown", e.Reason));
}
});
var partitionState = new Dictionary <TopicPartition, PartitionState>();
using (var producer = pBuilder.Build())
using (var consumer = cBuilder.Build())
{
consumer.Subscribe(InputTopic);
try
{
while (true)
{
ConsumeResult <TInKey, TInValue> cr;
try
{
cr = consumer.Consume(compositeCancellationToken);
}
catch (ConsumeException ex)
{
if (ex.Error.Code == ErrorCode.Local_ValueDeserialization)
{
// For an in-depth discussion of what to do in the event of deserialization errors, refer to:
// https://www.confluent.io/blog/kafka-connect-deep-dive-error-handling-dead-letter-queues
if (ConsumeErrorTolerance == ErrorTolerance.All)
{
continue;
}
errorCts.Cancel(); // no error tolerance.
}
Thread.Sleep(TimeSpan.FromSeconds(10)); // ?? if not fail fast, do we want to sleep and why?
continue;
}
if (!partitionState.ContainsKey(cr.TopicPartition))
{
partitionState.Add(cr.TopicPartition, new PartitionState(this));
}
partitionState[cr.TopicPartition].HandleConsumedMessage(cr, consumer, producer, funcExecSemaphore, errorCts);
aMessageHasBeenProcessed = true;
}
}
catch (OperationCanceledException) { }
}
if (errorCts.IsCancellationRequested)
{
throw new Exception("error occured, and we're failing fast.");
}
}