public Task <DeliveryReport> Produce(byte[] payload, byte[] key = null, Int32 partition = RD_KAFKA_PARTITION_UA)
{
// Passes the TaskCompletionSource to the delivery report callback
// via the msg_opaque pointer
var deliveryCompletionSource = new TaskCompletionSource <DeliveryReport>();
var gch = GCHandle.Alloc(deliveryCompletionSource);
while (true)
{
if (handle.Produce(payload, key, partition, GCHandle.ToIntPtr(gch)) == 0)
{
// Successfully enqueued produce request
break;
}
ErrorCode err = LibRdKafka.last_error();
if (err == ErrorCode._QUEUE_FULL)
{
// Wait and retry
Task.Delay(TimeSpan.FromMilliseconds(50)).Wait();
}
else
{
gch.Free();
throw RdKafkaException.FromErr(err, "Could not produce message");
}
}
return(deliveryCompletionSource.Task);
}
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config,
out LibRdKafka.PartitionerCallback partitionerDelegate)
{
// PartitionerDelegate is an out parameter as its reference must be kept outside of Topic
// it may be called after topic is GC, and it may be different for different topics
// so we can't simply make it static here
this.producer = producer;
config = config ?? new TopicConfig();
config["produce.offset.report"] = "true";
IntPtr configPtr = config.handle.Dup();
if (config.CustomPartitioner != null)
{
partitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
byte[] key = null;
if (keydata != IntPtr.Zero)
{
key = new byte[(int)keylen];
Marshal.Copy(keydata, key, 0, (int)keylen);
}
return(config.CustomPartitioner(this, key, partition_cnt));
};
LibRdKafka.topic_conf_set_partitioner_cb(configPtr, partitionerDelegate);
}
else
{
partitionerDelegate = null;
}
handle = kafkaHandle.Topic(topic, configPtr);
}
private void Produce(
string topic,
byte[] val, int valOffset, int valLength,
byte[] key, int keyOffset, int keyLength,
long?timestamp,
Int32 partition, bool blockIfQueueFull,
IDeliveryHandler deliveryHandler)
{
SafeTopicHandle topicHandle = getKafkaTopicHandle(topic);
if (!this.disableDeliveryReports && deliveryHandler != null)
{
// Passes the TaskCompletionSource to the delivery report callback via the msg_opaque pointer
var deliveryCompletionSource = deliveryHandler;
var gch = GCHandle.Alloc(deliveryCompletionSource);
var ptr = GCHandle.ToIntPtr(gch);
if (topicHandle.Produce(val, valOffset, valLength, key, keyOffset, keyLength, partition, timestamp, ptr, blockIfQueueFull) != 0)
{
var err = LibRdKafka.last_error();
gch.Free();
throw new KafkaException(err, "Could not produce message");
}
return;
}
if (topicHandle.Produce(val, valOffset, valLength, key, keyOffset, keyLength, partition, timestamp, IntPtr.Zero, blockIfQueueFull) != 0)
{
var err = LibRdKafka.last_error();
throw new KafkaException(err, "Could not produce message");
}
return;
}
private void Produce(byte[] payload, byte[] key, Int32 partition, object deliveryHandler)
{
var gch = GCHandle.Alloc(deliveryHandler);
var ptr = GCHandle.ToIntPtr(gch);
while (true)
{
if (handle.Produce(payload, key, partition, ptr) == 0)
{
// Successfully enqueued produce request
break;
}
var err = LibRdKafka.last_error();
if (err == ErrorCode._QUEUE_FULL)
{
// Wait and retry
Task.Delay(TimeSpan.FromMilliseconds(50)).Wait();
}
else
{
gch.Free();
throw RdKafkaException.FromErr(err, "Could not produce message");
}
}
}
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config)
{
this.producer = producer;
config = config ?? new TopicConfig();
config["produce.offset.report"] = "true";
IntPtr configPtr = config.handle.Dup();
if (config.CustomPartitioner != null)
{
PartitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
byte[] key = null;
if (keydata != IntPtr.Zero)
{
key = new byte[(int)keylen];
Marshal.Copy(keydata, key, 0, (int)keylen);
}
return(config.CustomPartitioner(this, key, partition_cnt));
};
LibRdKafka.topic_conf_set_partitioner_cb(configPtr, PartitionerDelegate);
}
handle = kafkaHandle.Topic(topic, configPtr);
}
/// <summary>
/// Initializes a new Producer instance.
/// </summary>
/// <param name="config">
/// librdkafka configuration parameters (refer to https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)
/// TODO: Link to confluent-kafka-dotnet page with dotnet specific parameters also (i.e. default.topic.config).
/// </param>
/// <param name="manualPoll">
/// If true, does not start a dedicated polling thread to trigger events or receive delivery reports -
/// you must call the Poll method periodically instead.
/// </param>
/// <param name="disableDeliveryReports">
/// If true, disables notification of delivery reports. Note: if set to true and you use a ProduceAsync variant that return
/// a Task, the Tasks will never complete. Generally you should leave this parameter as false. Set it to true for "fire and
/// forget" semantics and a small boost in performance.
/// </param>
public Producer(IEnumerable <KeyValuePair <string, object> > config, bool manualPoll = false, bool disableDeliveryReports = false)
{
this.topicConfig = (IEnumerable <KeyValuePair <string, object> >)config.FirstOrDefault(prop => prop.Key == "default.topic.config").Value;
this.manualPoll = manualPoll;
this.disableDeliveryReports = disableDeliveryReports;
var configHandle = SafeConfigHandle.Create();
config
.Where(prop => prop.Key != "default.topic.config")
.ToList()
.ForEach((kvp) => { configHandle.Set(kvp.Key, kvp.Value.ToString()); });
IntPtr configPtr = configHandle.DangerousGetHandle();
if (!disableDeliveryReports)
{
LibRdKafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback);
}
// TODO: provide some mechanism whereby calls to the error and log callbacks are cached until
// such time as event handlers have had a chance to be registered.
LibRdKafka.conf_set_error_cb(configPtr, ErrorCallback);
LibRdKafka.conf_set_log_cb(configPtr, LogCallback);
LibRdKafka.conf_set_stats_cb(configPtr, StatsCallback);
this.kafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr);
if (!manualPoll)
{
callbackCts = new CancellationTokenSource();
callbackTask = StartPollTask(callbackCts.Token);
}
}
/// <summary>
/// Wait for all rdkafka objects to be destroyed.
///
/// Returns if all kafka objects are now destroyed,
/// or throws TimeoutException if the timeout was reached.
///
/// Since RdKafka handle deletion is an async operation the
/// WaitDestroyed() function can be used for applications where
/// a clean shutdown is required.
/// </summary>
/// <exception cref="System.TimeoutException">Timeout was reached before all objects were destroyed.</exception>
public static void WaitDestroyed(TimeSpan timeout)
{
if ((long)LibRdKafka.wait_destroyed((IntPtr)timeout.TotalMilliseconds) != 0)
{
throw new TimeoutException();
}
}
/// <param name="opaque">
/// note: this property is set to that defined in rd_kafka_conf
/// (which is never used by confluent-kafka-dotnet).
/// </param>
private static void DeliveryReportCallbackImpl(IntPtr rk, IntPtr rkmessage, IntPtr opaque)
{
var msg = Marshal.PtrToStructure <rd_kafka_message>(rkmessage);
// the msg._private property has dual purpose. Here, it is an opaque pointer set
// by Topic.Produce to be an IDeliveryHandler. When Consuming, it's for internal
// use (hence the name).
if (msg._private == IntPtr.Zero)
{
// Note: this can occur if the ProduceAsync overload that accepts a DeliveryHandler
// was used and the delivery handler was set to null.
return;
}
var gch = GCHandle.FromIntPtr(msg._private);
var deliveryHandler = (IDeliveryHandler)gch.Target;
gch.Free();
byte[] key = null;
byte[] val = null;
if (deliveryHandler.MarshalData)
{
if (msg.key != IntPtr.Zero)
{
key = new byte[(int)msg.key_len];
System.Runtime.InteropServices.Marshal.Copy(msg.key, key, 0, (int)msg.key_len);
}
if (msg.val != IntPtr.Zero)
{
val = new byte[(int)msg.len];
System.Runtime.InteropServices.Marshal.Copy(msg.val, val, 0, (int)msg.len);
}
}
IntPtr timestampType;
long timestamp = LibRdKafka.message_timestamp(rkmessage, out timestampType) / 1000;
var dateTime = new DateTime(0);
if ((TimestampType)timestampType != TimestampType.NotAvailable)
{
dateTime = Timestamp.UnixTimestampMsToDateTime(timestamp);
}
deliveryHandler.HandleDeliveryReport(
new Message(
// TODO: tracking handle -> topicName in addition to topicName -> handle could
// avoid this marshalling / memory allocation cost.
Util.Marshal.PtrToStringUTF8(LibRdKafka.topic_name(msg.rkt)),
msg.partition,
msg.offset,
key,
val,
new Timestamp(dateTime, (TimestampType)timestampType),
msg.err
)
);
}
private void Produce(byte[] payload, int payloadCount, byte[] key, int keyCount, Int32 partition, object deliveryHandler, bool blockIfQueueFull)
{
var gch = GCHandle.Alloc(deliveryHandler);
var ptr = GCHandle.ToIntPtr(gch);
if (handle.Produce(payload, payloadCount, key, keyCount, partition, ptr, blockIfQueueFull) != 0)
{
var err = LibRdKafka.last_error();
gch.Free();
throw RdKafkaException.FromErr(err, "Could not produce message");
}
}
private static void ProcessInMemoryCertificatesIfConfigured(
SafeConfigHandle configHandle,
IntPtr configPtr,
ManualConfigSettings ManualConfigSettings)
{
if (ManualConfigSettings.X509Certificate == null)
{
return;
}
X509Certificate2 caCertificate = ManualConfigSettings.CAX509Certificate;
if (caCertificate == null)
{
caCertificate = AttemptToResolveCACertificateFromUserCertificate(ManualConfigSettings.X509Certificate);
}
if (caCertificate == null)
{
throw new InvalidOperationException("An accompanying in-memory CA certificate must be provided");
}
PrivateKeyAlgorithmAndBytes privateKeyAlgorithmAndBytes = ManualConfigSettings.PrivateKeyAlgorithmAndBytes;
if (privateKeyAlgorithmAndBytes == null)
{
privateKeyAlgorithmAndBytes = AttemptToResolvePrivateKeyFromUserCertificate(ManualConfigSettings.X509Certificate);
}
if (privateKeyAlgorithmAndBytes == null)
{
throw new InvalidOperationException("A private key must accompany the in-memory certificate");
}
LibRdKafka.conf_set_bytes(
configPtr,
ManualConfigSettings.SettingNameSslCertificateLocationInMemory,
ManualConfigSettings.X509Certificate.RawData);
LibRdKafka.conf_set_bytes(
configPtr,
ManualConfigSettings.SettingNameSslCALocationInMemory,
caCertificate.RawData);
LibRdKafka.conf_set_bytes(
configPtr,
ManualConfigSettings.SettingNameSslKeyInMemory,
privateKeyAlgorithmAndBytes.RawData);
configHandle.Set(
ManualConfigSettings.SettingNameSslKeyInMemoryType,
privateKeyAlgorithmAndBytes.PrivateKeyAlgorithmInUse);
}
private void LogCallback(IntPtr rk, int level, string fac, string buf)
{
var name = Util.Marshal.PtrToStringUTF8(LibRdKafka.name(rk));
if (OnLog == null)
{
// Log to stderr by default if no logger is specified.
Loggers.ConsoleLogger(this, new LogMessage(name, level, fac, buf));
return;
}
OnLog.Invoke(this, new LogMessage(name, level, fac, buf));
}
public Producer(Config config, string brokerList = null)
{
config = config ?? new Config();
IntPtr cfgPtr = config.handle.Dup();
LibRdKafka.conf_set_dr_msg_cb(cfgPtr, DeliveryReportDelegate);
Init(RdKafkaType.Producer, cfgPtr, config.Logger);
if (brokerList != null)
{
handle.AddBrokers(brokerList);
}
}
internal void Init(RdKafkaType type, IntPtr config, Config.LogCallback logger)
{
ErrorDelegate = (IntPtr rk, ErrorCode err, string reason, IntPtr opaque) =>
{
OnError?.Invoke(this, new ErrorArgs()
{
ErrorCode = err,
Reason = reason
});
};
LibRdKafka.conf_set_error_cb(config, ErrorDelegate);
if (logger == null)
{
logger = ((string handle, int level, string fac, string buf) =>
{
var now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff");
Console.WriteLine($"{level}|{now}|{handle}|{fac}| {buf}");
});
}
LogDelegate = (IntPtr rk, int level, string fac, string buf) =>
{
// The log_cb is called very early during construction, before
// SafeKafkaHandle or any of the C# wrappers are ready.
// So we can't really pass rk on, just pass the rk name instead.
var name = Marshal.PtrToStringAnsi(LibRdKafka.name(rk));
logger(name, level, fac, buf);
};
LibRdKafka.conf_set_log_cb(config, LogDelegate);
StatsDelegate = (IntPtr rk, IntPtr json, UIntPtr json_len, IntPtr opaque) =>
{
OnStatistics?.Invoke(this, Marshal.PtrToStringAnsi(json));
return(0);
};
LibRdKafka.conf_set_stats_cb(config, StatsDelegate);
handle = SafeKafkaHandle.Create(type, config);
callbackCts = new CancellationTokenSource();
callbackTask = StartCallbackTask(callbackCts.Token);
}
public Consumer(Config config, string brokerList = null)
{
RebalanceDelegate = RebalanceCallback;
CommitDelegate = CommitCallback;
IntPtr cfgPtr = config.handle.Dup();
LibRdKafka.conf_set_rebalance_cb(cfgPtr, RebalanceDelegate);
LibRdKafka.conf_set_offset_commit_cb(cfgPtr, CommitDelegate);
if (config.DefaultTopicConfig != null)
{
LibRdKafka.conf_set_default_topic_conf(cfgPtr,
config.DefaultTopicConfig.handle.Dup());
}
Init(RdKafkaType.Consumer, cfgPtr, config.Logger);
if (brokerList != null)
{
handle.AddBrokers(brokerList);
}
}
internal static string ErrorToString(ErrorCode errorCode) => Marshal.PtrToStringAnsi(LibRdKafka.err2str(errorCode));
/// <summary> /// Loads the native librdkafka library from the specified path (note: the /// specified path needs to include the filename). Does nothing if the /// library is already loaded. /// </summary> /// <returns> /// true if librdkafka was loaded as a result of this call, false if the /// library has already been loaded. /// </returns> /// <remarks> /// You will not typically need to call this method - librdkafka is loaded /// automatically on first use of a Producer or Consumer instance. /// </remarks> public static bool Load(string path) => LibRdKafka.Initialize(path);
public static void SetLogLevel(int logLevel)
{
LibRdKafka.set_log_level(IntPtr.Zero, (IntPtr)logLevel);
}
